Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore(libsy): reorg libsy mods (#100)
Greg Clark8a6858b12f29
diff --git a/Cargo.lock b/Cargo.lockindex 0ed7ec265..b432a6d95 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -857,26 +857,13 @@ version = "0.2.186"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"-[[package]]-name = "libsy"-version = "0.1.0"-dependencies = [- "async-trait",- "futures",- "rand 0.8.7",- "serde_json",- "switchyard-protocol",- "tokio",- "tokio-stream",-]-[[package]]name = "libsy-examples"version = "0.1.0"dependencies = ["async-trait","futures",- "libsy",+ "switchyard-libsy","switchyard-protocol","tokio","tokio-stream",@@ -1629,6 +1616,19 @@ dependencies = ["tracing",]+[[package]]+name = "switchyard-libsy"+version = "0.1.0"+dependencies = [+ "async-trait",+ "futures",+ "rand 0.8.7",+ "serde_json",+ "switchyard-protocol",+ "tokio",+ "tokio-stream",+]+[[package]]name = "switchyard-protocol"version = "0.1.0"@@ -1645,7 +1645,6 @@ version = "0.1.0"dependencies = ["async-trait","futures-util",- "libsy","pyo3","pyo3-async-runtimes","pythonize",@@ -1654,6 +1653,7 @@ dependencies = ["switchyard-components","switchyard-components-v2","switchyard-core",+ "switchyard-libsy","switchyard-server","switchyard-translation","tokio",diff --git a/Cargo.toml b/Cargo.tomlindex 582ea8dfe..ec2ee2c7a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,7 +11,7 @@ members = ["crates/switchyard-components-v2-macros","crates/switchyard-core","crates/switchyard-py",- "crates/libsy-protocol",+ "crates/protocol","crates/switchyard-server","crates/switchyard-skill-distillation","crates/switchyard-translation",diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.tomlindex 82643f8b6..2f9ccfe55 100644--- a/crates/libsy-examples/Cargo.toml+++ b/crates/libsy-examples/Cargo.toml@@ -13,8 +13,8 @@ repository.workspace = truerust-version.workspace = true[dependencies]-libsy = { path = "../libsy" }-switchyard-protocol = { path = "../libsy-protocol" }+switchyard-libsy = { path = "../libsy" }+switchyard-protocol = { path = "../protocol" }async-trait = "0.1"futures = "0.3"tokio = { version = "1", features = ["full"] }diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rsindex 102bcb525..c151ee48e 100644--- a/crates/libsy-examples/src/ensemble.rs+++ b/crates/libsy-examples/src/ensemble.rs@@ -22,7 +22,7 @@ use std::sync::{Arc, Mutex};use async_trait::async_trait;-use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};+use switchyard_libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};use switchyard_protocol::{completion_text, prompt_text, text_request};/// Which step of the ensemble flow produced a decision.@@ -100,8 +100,8 @@ impl EnsembleOrchAlgo {/// exploring for `exploration_turns` before committing to the winningest/// candidate (`0` = never commit, ensemble every request), routing among/// `target_set`. Wrap it in an [`Arc`](std::sync::Arc) and drive it with- /// [`run`](libsy::Algorithm::run) or- /// [`run_stream`](libsy::Algorithm::run_stream).+ /// [`run`](switchyard_libsy::Algorithm::run) or+ /// [`run_stream`](switchyard_libsy::Algorithm::run_stream).pub fn new(candidate_models: Vec<String>,judge_model: impl Into<String>,@@ -395,8 +395,8 @@ impl Algorithm for EnsembleOrchAlgo {#[cfg(test)]mod tests {use super::*;- use libsy::{LlmResponse, LlmTarget, Response, RoutedLlmClient, Signals};use std::sync::Mutex as StdMutex;+ use switchyard_libsy::{LlmResponse, LlmTarget, Response, RoutedLlmClient, Signals};use switchyard_protocol::text_response;/// Mock client that answers candidate calls with `answer from {model}` and,diff --git a/crates/libsy-examples/src/lib.rs b/crates/libsy-examples/src/lib.rsindex a3c9ff554..505dcc9f8 100644--- a/crates/libsy-examples/src/lib.rs+++ b/crates/libsy-examples/src/lib.rs@@ -3,8 +3,8 @@//! Reference algorithms for [`libsy`], kept out of the core crate but compiled and//! tested here so they stay current. Each is a worked example of the-//! [`Algorithm`](libsy::Algorithm) trait; the `examples/` directory has runnable agents-//! that drive them. The core crate provides [`libsy::RandomAlgo`] for uniform random+//! [`Algorithm`](switchyard_libsy::Algorithm) trait; the `examples/` directory has runnable agents+//! that drive them. The core crate provides [`switchyard_libsy::algorithms::Random`] for uniform random//! routing.//!//! - [`ensemble::EnsembleOrchAlgo`] — fan out to several models, judge, and commitdiff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex df8303835..911169e52 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -2,7 +2,7 @@# SPDX-License-Identifier: Apache-2.0[package]-name = "libsy"+name = "switchyard-libsy"version = "0.1.0"description = "Switchyard library crate"authors.workspace = true@@ -16,7 +16,7 @@ async-trait = "0.1"serde_json = "1"futures = "0.3"rand = "0.8"-switchyard-protocol = { path = "../libsy-protocol" }+switchyard-protocol = { path = "../protocol" }tokio = { version = "1", features = ["full"] }tokio-stream = "0.1"diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 3282bc240..40a736f86 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -11,8 +11,8 @@ so it drops into a proxy, gateway, or agent runtime.Build a target set, pick an algorithm, run a request:```rust-use libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request};-use libsy::LlmClassifierOrchAlgo;+use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request};+use switchyard_libsy::algorithms::LlmClassifier;use switchyard_protocol::{completion_text, text_request};use std::sync::Arc;@@ -20,7 +20,7 @@ use std::sync::Arc;let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn RoutedLlmClient>;let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };-let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(+let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new("classifier", "strong", "weak", 0.5,LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]),));@@ -35,7 +35,7 @@ let req = Request {let (trace, response) = algo.clone().run(Context::default(), req).await?; // calls in, trace + response outprintln!("routed to {}", trace.last().unwrap().selected_model());// `response.llm_response` is buffered or streamed; fold it to the aggregate to read text.-println!("answer: {}", completion_text(&response.llm_response.aggregate().await?));+println!("answer: {}", completion_text(&response.llm_response.into_agg().await?));```Runnable: [`research_agent`](examples/research_agent.rs)@@ -152,17 +152,17 @@ match response.llm_response {}```-Need the whole answer regardless of how it arrived? `aggregate()` folds a `Stream` (or+Need the whole answer regardless of how it arrived? `into_agg()` folds a `Stream` (orreturns an `Agg` unchanged) into a single `AggLlmResponse`, surfacing any mid-stream error:```rust-let agg = response.llm_response.aggregate().await?; // drives the stream to completion+let agg = response.llm_response.into_agg().await?; // drives the stream to completionprintln!("{}", completion_text(&agg));```An `LlmResponseChunk` is the provider-neutral streaming event (`MessageStart`, `TextDelta`,`ReasoningDelta`, `ToolCallDelta`, `Usage`, `MessageStop`, `Error`); `ResponseAccumulator`-is the fold behind `aggregate()` if you need to assemble one yourself. Runnable end-to-end+is the fold behind `into_agg()` if you need to assemble one yourself. Runnable end-to-enddemo: [`streaming_agent`](examples/streaming_agent.rs).## Driving the calls yourself (`run_stream`)@@ -221,7 +221,7 @@ Example — the LLM classifier (classify, then route; full version in```rust#[async_trait]-impl Algorithm for LlmClassifierOrchAlgo {+impl Algorithm for LlmClassifier {async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)-> Result<Response, Box<dyn Error + Send + Sync>> {// Thread `ctx` into every offloaded call and decision — it carries the request's@@ -233,8 +233,8 @@ impl Algorithm for LlmClassifierOrchAlgo {let classify_response =driver.call_llm_target(ctx.clone(), &classifier, classify_req, classify_decision).await?;// Abbreviated: fold the (buffered or streamed) response to its aggregate and read- // the completion text as a score. `.agg()` alone is `None` for an unfolded stream.- let score = classify_response.llm_response.aggregate().await+ // the completion text as a score. `.as_agg()` alone is `None` for an unfolded stream.+ let score = classify_response.llm_response.into_agg().await.ok().and_then(|agg| completion_text(&agg).trim().parse::<f64>().ok());@@ -256,9 +256,9 @@ agents live in [`examples`](examples/) folder.**Reference algorithms** — implementations to read and route with:-- [`RandomAlgo`](src/algorithms/rand.rs) — uniform random over the set+- [`Random`](src/algorithms/rand.rs) — uniform random over the set(one call).-- [`LlmClassifierOrchAlgo`](src/algorithms/llm_class.rs) — classify, then route+- [`LlmClassifier`](src/algorithms/llm_class.rs) — classify, then routestrong/weak; fail open to strong.- [`EnsembleOrchAlgo`](../libsy-examples/src/ensemble.rs) — stateful: fan out tocandidates, judge the best, commit to the winner after N exploration turns.diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rsindex eabfb9bda..f3fd2f873 100644--- a/crates/libsy/examples/research_agent.rs+++ b/crates/libsy/examples/research_agent.rs@@ -15,8 +15,8 @@ use std::error::Error;use std::sync::Arc;use async_trait::async_trait;-use libsy::LlmClassifierOrchAlgo;-use libsy::{+use switchyard_libsy::algorithms::LlmClassifier;+use switchyard_libsy::{Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response,RoutedLlmClient,};@@ -91,14 +91,9 @@ impl ResearchAgent {#[tokio::main(flavor = "current_thread")]async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {// Configure routing once: an LLM classifier over three named targets. Swapping- // in `RandomAlgo` needs no change to the agent.- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(- CLASSIFIER,- STRONG,- WEAK,- 0.5,- targets(),- ));+ // in `Random` needs no change to the agent.+ let algo: Arc<dyn Algorithm> =+ Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));let agent = ResearchAgent { algo };println!("{}", agent.run("what is switchyard?").await?);diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rsindex 0199728bb..64e2dbac8 100644--- a/crates/libsy/examples/research_agent_core.rs+++ b/crates/libsy/examples/research_agent_core.rs@@ -12,8 +12,8 @@use std::error::Error;use std::sync::Arc;-use libsy::LlmClassifierOrchAlgo;-use libsy::{+use switchyard_libsy::algorithms::LlmClassifier;+use switchyard_libsy::{Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request, Response, Step,};use switchyard_protocol::{completion_text, text_request, text_response};@@ -98,13 +98,8 @@ fn print_decision(decision: &dyn Decision) {#[tokio::main(flavor = "current_thread")]async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(- CLASSIFIER,- STRONG,- WEAK,- 0.5,- targets(),- ));+ let algo: Arc<dyn Algorithm> =+ Arc::new(LlmClassifier::new(CLASSIFIER, STRONG, WEAK, 0.5, targets()));let mut agent = ResearchAgent { algo };println!("{}", agent.run("what is switchyard?").await?);diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rsindex 5c245a8b9..7038a33e2 100644--- a/crates/libsy/examples/streaming_agent.rs+++ b/crates/libsy/examples/streaming_agent.rs@@ -17,9 +17,10 @@ use std::io::Write;use std::sync::Arc;use futures::StreamExt;-use libsy::{+use switchyard_libsy::algorithms::Random;+use switchyard_libsy::{Algorithm, Context, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmTarget, LlmTargetSet,- RandomAlgo, Request, Response, Step,+ Request, Response, Step,};use switchyard_protocol::{completion_text, text_request};@@ -55,7 +56,7 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {semantic_name: "stream/model".to_string(),llm_client: None,}]);- let algo: Arc<dyn Algorithm> = Arc::new(RandomAlgo::new(targets));+ let algo: Arc<dyn Algorithm> = Arc::new(Random::new(targets));let request = Request {llm_request: text_request(Some("auto".to_string()), "tell me about switchyard"),diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rsindex ce17da02b..6288e4aec 100644--- a/crates/libsy/src/algorithms.rs+++ b/crates/libsy/src/algorithms.rs@@ -1,6 +1,15 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0+//! Concrete algorithms and the interfaces for building them.+//!+//! Reach for them by name — `use switchyard_libsy::algorithms::Random` — rather than through the+//! per-algorithm submodules.+pub mod llm_class;pub mod noop;pub mod rand;++pub use llm_class::{ClassifierDecision, ClassifierTier, LlmClassifier};+pub use noop::{Noop, NoopDecision};+pub use rand::{Random, RandomDecision};diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex e3fe7ebf0..aa10a9fef 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -64,7 +64,7 @@ impl Decision for ClassifierDecision {}/// LLM-classifier router: classify with one target, then route to strong/weak.-pub struct LlmClassifierOrchAlgo {+pub struct LlmClassifier {classifier_model: String,strong_model: String,weak_model: String,@@ -72,7 +72,7 @@ pub struct LlmClassifierOrchAlgo {target_set: LlmTargetSet,}-impl LlmClassifierOrchAlgo {+impl LlmClassifier {/// Configure the classifier: the model that scores each request, the strong/// and weak models to route to, the score `threshold` at or above which the/// strong model is chosen, and the `target_set` to route among. That set must@@ -138,7 +138,7 @@ fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec<Message}#[async_trait]-impl Algorithm for LlmClassifierOrchAlgo {+impl Algorithm for LlmClassifier {async fn create_run_task(self: Arc<Self>,ctx: Context,@@ -251,7 +251,7 @@ mod tests {}/// Build a classifier algo whose three targets share a scoring client.- fn algo(threshold: f64, score: &str) -> (LlmClassifierOrchAlgo, Arc<Mutex<Vec<Request>>>) {+ fn algo(threshold: f64, score: &str) -> (LlmClassifier, Arc<Mutex<Vec<Request>>>) {let seen = Arc::new(Mutex::new(Vec::new()));let client = Arc::new(ScoringClient {classifier_model: "router/classifier".to_string(),@@ -267,7 +267,7 @@ mod tests {target("frontier/model"),target("cheap/model"),]);- let algo = LlmClassifierOrchAlgo {+ let algo = LlmClassifier {classifier_model: "router/classifier".to_string(),strong_model: "frontier/model".to_string(),weak_model: "cheap/model".to_string(),@@ -286,7 +286,7 @@ mod tests {}/// Wrap a classifier algo as `Arc<dyn Algorithm>` we can drive to completion.- fn orch(algo: LlmClassifierOrchAlgo) -> Arc<dyn Algorithm> {+ fn orch(algo: LlmClassifier) -> Arc<dyn Algorithm> {Arc::new(algo)}diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex cefab1165..a3ec368cd 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -13,9 +13,9 @@ use switchyard_protocol::{use crate::{Algorithm, Context, Decision, Driver};/// A routing algorithm that does not route. It returns a hard-coded response.-pub struct NoopAlgo {}+pub struct Noop {}-/// How [`NoopAlgo`] records which model it chose. This will be the model on the Request if any,+/// How [`Noop`] records which model it chose. This will be the model on the Request if any,/// otherwise a hard coded placeholder. Neither is actually used.pub struct NoopDecision {model: String,@@ -34,7 +34,7 @@ impl Decision for NoopDecision {}#[async_trait::async_trait]-impl Algorithm for NoopAlgo {+impl Algorithm for Noop {async fn create_run_task(self: Arc<Self>,ctx: Context,@@ -89,7 +89,7 @@ mod tests {metadata: None,};- let a: Arc<dyn Algorithm> = Arc::new(NoopAlgo {});+ let a: Arc<dyn Algorithm> = Arc::new(Noop {});let (decisions, response) = a.run(Context::default(), request).await?;let Some(decision) = decisions.first() else {panic!("Expected exactly one Decision");diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex adc2b366e..817fa9566 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -16,7 +16,7 @@ use rand::seq::SliceRandom;use crate::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response};-/// Decision produced by [`RandomAlgo`]: which target was chosen and why.+/// Decision produced by [`Random`]: which target was chosen and why.pub struct RandomDecision {/// The randomly selected target/model.pub selected_model: String,@@ -39,11 +39,11 @@ impl Decision for RandomDecision {}/// Uniform random router over a target set.-pub struct RandomAlgo {+pub struct Random {target_set: LlmTargetSet,}-impl RandomAlgo {+impl Random {/// Creates a router over `target_set`.////// Wrap it in an [`Arc`] and drive it with [`Algorithm::run`] or@@ -54,7 +54,7 @@ impl RandomAlgo {}#[async_trait]-impl Algorithm for RandomAlgo {+impl Algorithm for Random {async fn create_run_task(self: Arc<Self>,ctx: Context,@@ -120,7 +120,7 @@ mod tests {}/// Builds a random router whose targets all share an echo client.- fn algorithm(names: &[&str]) -> RandomAlgo {+ fn algorithm(names: &[&str]) -> Random {let targets = names.iter().map(|name| LlmTarget {@@ -128,7 +128,7 @@ mod tests {llm_client: Some(Arc::new(EchoClient)),}).collect();- RandomAlgo::new(LlmTargetSet::new(targets))+ Random::new(LlmTargetSet::new(targets))}fn shared_algorithm(names: &[&str]) -> Arc<dyn Algorithm> {diff --git a/crates/libsy/src/core.rs b/crates/libsy/src/core.rsnew file mode 100644index 000000000..a46cf2f4c--- /dev/null+++ b/crates/libsy/src/core.rs@@ -0,0 +1,12 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Core orchestration: the [`Algorithm`] trait and its [`Driver`], built on the+//! type-erased promise-over-a-stream pump in [`driver`]. Its public items are+//! re-exported at the crate root; algorithm implementations live in+//! [`crate::algorithms`].++pub mod algorithm;+mod driver;++pub use algorithm::*;diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsnew file mode 100644index 000000000..621fb980b--- /dev/null+++ b/crates/libsy/src/core/algorithm.rs@@ -0,0 +1,931 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! The [`Algorithm`] trait and its [`Driver`] — the orchestration contract every+//! routing/optimization algorithm implements, and the offload channel it makes model+//! calls and publishes [`Decision`]s over. See the crate root for the narrative model.++use std::{error::Error, pin::Pin, sync::Arc};++use async_trait::async_trait;+use futures::{Stream, StreamExt};++/// The request/response protocol types, re-exported from [`switchyard_protocol`].+/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;+/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response+/// (a live [`LlmResponseStream`] or the terminal aggregate).+pub use switchyard_protocol::{+ AggLlmResponse, Context, Decision, LlmRequest, LlmResponse, LlmResponseChunk,+ LlmResponseStream, Metadata, Request, Response, RoutedLlmClient, Signals,+};++use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};++/// Shorthand for the crate's boxed, thread-safe error type.+type BoxErr = Box<dyn Error + Send + Sync>;++/// A boxed, `Send` stream of [`Step`]s — the output of+/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps+/// `Arc<dyn Algorithm>` object-safe.+pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;++/// A request paired with the routing [`Decision`] that produced it — the offload+/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.+///+/// The two model identifiers live in separate, unambiguous places: the model to+/// call is [`decision.selected_model()`](Decision::selected_model), while+/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy+/// never overwrites it). A client maps `selected_model()` to the provider model+/// id it hits.+#[derive(Clone)]+pub struct RoutedRequest {+ /// The request to serve; its `model` is the agent's original name.+ pub request: Request,+ /// The routing decision behind this call; `selected_model()` is the model to hit.+ pub decision: Arc<dyn Decision>,+ /// The client that serves this call by default, or `None` when the routed target+ /// had no client. Rides along on the offloaded call so a host driving the stream+ /// can serve it by default or override it with its own transport.+ pub default_client: Option<Arc<dyn RoutedLlmClient>>,+ /// The request's cross-cutting context, carried through the offload so whoever+ /// serves the call (libsy's own `run`, or a host driving the stream) hands it to+ /// [`RoutedLlmClient::call`].+ pub ctx: Context,+}++/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].+///+/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the+/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it+/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and+/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's+/// [`Driver::call_llm`] on the other side.+pub struct CallLlmRequest {+ inner: DriverRequest,+ routed: RoutedRequest,+}++impl CallLlmRequest {+ /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy+ /// so the accessors are plain field reads.+ fn new(inner: DriverRequest) -> Self {+ // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a+ // mismatch would be a libsy bug, not a runtime condition.+ let routed = match inner.request::<RoutedRequest>() {+ Ok(routed) => routed.clone(),+ Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),+ };+ Self { inner, routed }+ }++ /// The routed request the host should serve. Its+ /// [`default_client`](RoutedRequest::default_client) serves the call by default,+ /// and its `decision.selected_model()` names the model to hit.+ pub fn get_routed(&self) -> &RoutedRequest {+ &self.routed+ }++ /// The model request to perform (the [`Request`] inside the routed request).+ pub fn get_request(&self) -> &Request {+ &self.get_routed().request+ }++ /// The decision that led to this call — its `selected_model()` is the model to hit.+ pub fn get_decision(&self) -> &dyn Decision {+ self.get_routed().decision.as_ref()+ }++ /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to+ /// propagate a failed model call back to the algorithm. Consumes the promise: it+ /// can only be fulfilled once.+ pub fn respond(self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {+ self.inner.respond::<Response>(result)+ }+}++/// The offload channel handed to an algorithm's+/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls+/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and+/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the+/// request's [`Step`] stream and awaits the consumer's response. The step channel is+/// bounded, so the consumer paces the algorithm one step at a time.+#[derive(Clone)]+pub struct Driver {+ driver: TypeErasedDriver,+}++impl Driver {+ /// Build an empty driver with its step channel ready. Created per call by+ /// [`run_stream`](Algorithm::run_stream).+ pub(crate) fn new() -> Self {+ Self {+ driver: TypeErasedDriver::new(),+ }+ }++ /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the+ /// consumer's [`Response`]. The call's context travels inside+ /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.+ pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {+ self.driver+ .fulfill_request::<RoutedRequest, Response>(routed.ctx.clone(), routed)+ .await+ }++ /// Offload a call to `target`: pair `request` with `decision` and the target's+ /// default client into a [`RoutedRequest`], then publish it (see+ /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;+ /// `decision.selected_model()` names the model to hit, and `request`'s+ /// `model` is left untouched.+ pub async fn call_llm_target(+ &self,+ ctx: Context,+ target: &LlmTarget,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, BoxErr> {+ self.call_llm(RoutedRequest {+ request,+ decision,+ default_client: target.llm_client.clone(),+ ctx,+ })+ .await+ }++ /// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.+ pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {+ self.driver.info(ctx, decision).await+ }++ /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream+ /// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)+ /// when the algorithm finishes.+ pub(crate) async fn finish(+ &self,+ ctx: Context,+ result: Result<Response, BoxErr>,+ ) -> Result<(), BoxErr> {+ match result {+ Ok(response) => self.driver.done(ctx, response).await,+ Err(err) => self.driver.fail(ctx, err).await,+ }+ }++ /// Transform the raw driver stream into a stream of [`Step`]s. Internal: the+ /// consumer stream is taken (once) by [`run_stream`](Algorithm::run_stream). A+ /// payload that does not match the expected type for its step becomes an `Err` item.+ pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {+ self.driver.stream().map(|item| match item? {+ DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),+ DriverStep::Info(payload) => payload+ .downcast::<Arc<dyn Decision>>()+ .map(|decision| Step::Decision(*decision))+ .map_err(|_| "driver: info payload was not a Decision".into()),+ DriverStep::Done(payload) => payload+ .downcast::<Response>()+ .map(Step::ReturnToAgent)+ .map_err(|_| "driver: done payload was not a Response".into()),+ })+ }+}++impl Default for Driver {+ fn default() -> Self {+ Self::new()+ }+}++/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].+pub enum Step {+ /// The algorithm needs this model call performed. The host serves it (optionally+ /// via [`RoutedRequest::default_client`]) and fulfills it with+ /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.+ CallLlm(Box<CallLlmRequest>),+ /// A routing decision the algorithm made, published via [`Driver::info`] as it+ /// happens (rather than collected into a trace returned at the end).+ Decision(Arc<dyn Decision>),+ /// The algorithm finished with its final response — the last step of a run.+ ReturnToAgent(Box<Response>),+}++/// A named routing target: a `semantic_name` an algorithm routes by, and an optional+/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to+/// [`Driver::call_llm_target`]; the client rides along as+/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.+#[derive(Clone)]+pub struct LlmTarget {+ /// The routing label an algorithm selects this target by — a logical tier like+ /// `"strong"`, or the model id when they coincide. Mapping it to a provider model+ /// id is the client's concern, never the algorithm's.+ pub semantic_name: String,+ /// The client that serves this target's calls by default, or `None` (then the+ /// stream consumer must serve them).+ pub llm_client: Option<Arc<dyn RoutedLlmClient>>,+}++/// The set of targets an algorithm may route among. An algorithm is constructed+/// with one and picks targets by position ([`targets`](Self::targets)) or by name+/// ([`get_target`](Self::get_target)).+#[derive(Clone)]+pub struct LlmTargetSet {+ targets: Vec<LlmTarget>,+}++impl LlmTargetSet {+ /// Build a target set from a list of targets.+ pub fn new(targets: Vec<LlmTarget>) -> Self {+ Self { targets }+ }++ /// All targets in the set — e.g. for an algorithm to select among.+ pub fn targets(&self) -> &[LlmTarget] {+ &self.targets+ }++ /// Look up a target by name; errors if no target has that name.+ pub fn get_target(&self, name: &str) -> Result<LlmTarget, Box<dyn Error + Send + Sync>> {+ self.targets+ .iter()+ .find(|t| t.semantic_name == name)+ .cloned()+ .ok_or(format!("Target {} not found", name).into())+ }+}++/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);+/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)+/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).+///+/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across+/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms+/// (the reference routers) get this for free; a stateful one uses interior mutability+/// over just its own state.+#[async_trait]+pub trait Algorithm: Send + Sync + 'static {+ /// Run one request to completion: make model calls with [`Driver::call_llm_target`],+ /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].+ /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)+ /// drive it. `ctx` carries cross-cutting request state (empty today).+ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>>;++ /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The+ /// reference algorithms ignore signals; a stateful algorithm updates its own+ /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.+ #[allow(unused_variables)]+ async fn process_signals(+ self: Arc<Self>,+ signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ Ok(())+ }++ /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its+ /// own task; drive the stream: serve each [`Step::CallLlm`] (via its+ /// [`default_client`](RoutedRequest::default_client) or your own transport) and read+ /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is+ /// bounded, so pulling paces the algorithm; each call is independent, so many run+ /// concurrently.+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {+ // This call's own driver: take its consumer stream, hand a producer-side clone to+ // the algorithm task, and keep one to emit the terminal step. The task blocks+ // publishing a step until the consumer pulls the previous one.+ let driver = Driver::new();+ let stream = driver.stream();+ tokio::spawn(async move {+ let outcome = self+ .create_run_task(ctx.clone(), driver.clone(), request)+ .await;+ let _ = driver.finish(ctx, outcome).await;+ });+ Box::pin(stream)+ }++ /// Run one request to completion, serving each offloaded call with its+ /// [`RoutedRequest::default_client`], and return the decision trace plus the final+ /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,+ /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model+ /// clients and the host wants the answer (and the decisions behind it); drive+ /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned+ /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]+ /// or a live [`LlmResponse::Stream`]; the caller decides whether to+ /// [`into_agg`](LlmResponse::into_agg) it or forward its chunks. Errors if a routed+ /// target has no client to serve its call, or the algorithm fails.+ async fn run(+ self: Arc<Self>,+ ctx: Context,+ request: Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {+ // Serve up to this many offloaded calls concurrently, so an algorithm that+ // fans out (e.g. an ensemble) isn't serialized on the client.+ const MAX_CONCURRENT_CALLS: usize = 10;++ // Serve one offloaded call with its target's default client. A failed *model*+ // call is forwarded to the algorithm via `respond`; this errors only on an+ // infrastructure failure (no default client, or the promise was dropped).+ async fn serve(call: CallLlmRequest) -> Result<(), Box<dyn Error + Send + Sync>> {+ let routed = call.get_routed().clone();+ let client = routed.default_client.clone().ok_or_else(|| {+ format!(+ "run: target '{}' has no client to serve the call",+ routed.decision.selected_model()+ )+ })?;+ call.respond(+ client+ .call(routed.ctx, routed.request, routed.decision)+ .await,+ )+ }++ let stream = self.run_stream(ctx, request);+ tokio::pin!(stream);++ let mut trace: Vec<Arc<dyn Decision>> = Vec::new();+ let mut in_flight = futures::stream::FuturesUnordered::new();+ let mut final_response: Option<Response> = None;+ let mut stream_open = true;++ while stream_open || !in_flight.is_empty() {+ tokio::select! {+ // Surface a failed serve as soon as it completes.+ Some(result) = in_flight.next(), if !in_flight.is_empty() => result?,+ // Pull the next step, unless the stream ended or we're at the cap.+ step = stream.next(), if stream_open && in_flight.len() < MAX_CONCURRENT_CALLS => {+ match step {+ None => stream_open = false,+ Some(item) => match item? {+ Step::CallLlm(call) => in_flight.push(serve(*call)),+ Step::Decision(decision) => trace.push(decision),+ Step::ReturnToAgent(response) => {+ final_response = Some(*response);+ stream_open = false;+ }+ },+ }+ }+ }+ }++ // Return whatever the algorithm produced — buffered or streamed. The caller+ // decides whether to aggregate the response or forward its chunks.+ final_response+ .map(|response| (trace, response))+ .ok_or_else(|| "run: stream ended without a final response".into())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use futures::StreamExt;+ use switchyard_protocol::{completion_text, text_request, text_response};++ /// Mock client that echoes back the target name it was called with.+ struct EchoClient;++ #[async_trait]+ impl RoutedLlmClient for EchoClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ // Echo back the model the algorithm routed to (the decision's selection).+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ decision.selected_model().to_string(),+ )),+ metadata: None,+ })+ }+ }++ /// Trivial decision + algo used only to exercise the orchestrator: calls the+ /// first target and returns its response with a one-item trace.+ struct TestDecision {+ model: String,+ }++ impl Decision for TestDecision {+ fn selected_model(&self) -> &str {+ &self.model+ }+ fn reasoning(&self) -> Option<&str> {+ None+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+ }++ struct TestAlgo {+ target_set: LlmTargetSet,+ }++ #[async_trait]+ impl Algorithm for TestAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let target = self+ .target_set+ .targets()+ .first()+ .ok_or("no targets")?+ .clone();+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {+ model: target.semantic_name.clone(),+ });+ driver.info(ctx.clone(), decision.clone()).await?;+ driver+ .call_llm_target(ctx, &target, request, decision)+ .await+ }+ }++ /// Build a shared `TestAlgo` over the given target set.+ fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {+ Arc::new(TestAlgo { target_set })+ }++ fn request() -> Request {+ Request {+ llm_request: text_request(Some("auto".to_string()), "hi".to_string()),+ raw_request: None,+ metadata: None,+ }+ }++ /// `(name, has_client)` — `has_client: false` builds a target with no default client.+ fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {+ let targets = names+ .iter()+ .map(|(name, has_client)| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),+ })+ .collect();+ LlmTargetSet::new(targets)+ }++ /// Client that serves a call as a token stream — its `call` returns+ /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).+ struct StreamingClient {+ chunks: Vec<LlmResponseChunk>,+ }++ #[async_trait]+ impl RoutedLlmClient for StreamingClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ _decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();+ Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ })+ }+ }++ /// Build a single-target algo whose one target streams `chunks`.+ fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {+ let target = LlmTarget {+ semantic_name: "stream/model".to_string(),+ llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),+ };+ orch(LlmTargetSet::new(vec![target]))+ }++ #[tokio::test]+ async fn run_returns_a_streamed_response_the_caller_aggregates(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,+ // and `run` returns the live stream untouched for the caller to fold.+ let orch = streaming_orch(vec![+ LlmResponseChunk::MessageStart {+ id: Some("m1".to_string()),+ model: Some("stream/model".to_string()),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "hel".to_string(),+ },+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "lo".to_string(),+ },+ LlmResponseChunk::MessageStop {+ reason: Some("stop".to_string()),+ },+ ]);+ let (trace, response) = orch.run(Context::default(), request()).await?;+ // `run` handed back the live stream; the caller folds it to a buffered aggregate.+ let agg = response.llm_response.into_agg().await?;+ assert_eq!(completion_text(&agg), "hello");+ assert_eq!(agg.model.as_deref(), Some("stream/model"));+ assert_eq!(trace.len(), 1);+ Ok(())+ }++ #[tokio::test]+ async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only+ // when the caller aggregates it.+ let orch = streaming_orch(vec![+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "partial".to_string(),+ },+ LlmResponseChunk::Error {+ message: "upstream exploded".to_string(),+ },+ ]);+ let (_, response) = orch.run(Context::default(), request()).await?;+ match response.llm_response.into_agg().await {+ Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),+ Err(err) => {+ assert!(err.to_string().contains("upstream exploded"));+ Ok(())+ }+ }+ }++ #[tokio::test]+ async fn run_offloads_via_promise_then_returns_to_agent(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target -> its call is offloaded via a promise the+ // orchestrator surfaces as a `CallLlm` step for us to fulfill.+ let stream =+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut saw_call = false;+ let mut final_completion = None;+ while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ saw_call = true;+ // The decision rode along with the promise.+ assert_eq!(call.get_decision().selected_model(), "offload/model");+ // Fulfilling the promise is the "real" model call the caller makes.+ call.respond(Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ "fulfilled".to_string(),+ )),+ metadata: None,+ }))?;+ }+ Step::Decision(decision) => {+ assert_eq!(decision.selected_model(), "offload/model");+ }+ Step::ReturnToAgent(response) => {+ final_completion = Some(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ );+ }+ }+ }++ assert!(saw_call, "expected a CallLlm step before ReturnToAgent");+ assert_eq!(+ final_completion.ok_or("no ReturnToAgent step")?,+ "fulfilled"+ );+ Ok(())+ }++ #[tokio::test]+ async fn client_backed_target_offloads_with_a_default_client(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Every call now offloads to the stream; a client-backed target rides its+ // client along as `default_client` so the consumer can serve it by default.+ let stream =+ orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut final_completion = None;+ while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ let routed = call.get_routed().clone();+ let client = routed+ .default_client+ .clone()+ .ok_or("expected a default client")?;+ let result = client+ .call(routed.ctx, routed.request, routed.decision)+ .await;+ call.respond(result)?;+ }+ Step::Decision(_) => {}+ Step::ReturnToAgent(response) => {+ final_completion = Some(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ );+ }+ }+ }++ // EchoClient echoes the model name back as the completion.+ assert_eq!(final_completion.ok_or("no ReturnToAgent")?, "direct/model");+ Ok(())+ }++ #[tokio::test]+ async fn run_returns_the_response_when_all_targets_have_clients(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Every target has a client, so run serves every call via the+ // default client and returns the trace + final response.+ let (trace, response) = orch(target_set(&[("direct/model", true)]))+ .run(Context::default(), request())+ .await?;+ // TestAlgo calls the first target; EchoClient echoes its name.+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "direct/model"+ );+ assert_eq!(trace[0].selected_model(), "direct/model");+ Ok(())+ }++ #[tokio::test]+ async fn run_errors_when_a_target_lacks_a_client() -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target has no default client to serve its offloaded call, so+ // driving it to completion errors.+ assert!(orch(target_set(&[("offload/model", false)]))+ .run(Context::default(), request())+ .await+ .is_err());+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 12)]+ async fn requests_are_processed_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {+ use std::time::Duration;+ use tokio::sync::Barrier;++ const N: usize = 12;++ // A client that blocks until all N concurrent calls have arrived. If+ // requests were serialized (one algorithm behind a `Mutex`), only one+ // call could be in flight, the barrier would never reach N, and the test+ // would time out. It passes only because the shared algorithm is driven+ // concurrently across requests.+ struct BarrierClient {+ barrier: Arc<Barrier>,+ }++ #[async_trait]+ impl RoutedLlmClient for BarrierClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ self.barrier.wait().await;+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ decision.selected_model().to_string(),+ )),+ metadata: None,+ })+ }+ }++ let barrier = Arc::new(Barrier::new(N));+ let targets = LlmTargetSet::new(vec![LlmTarget {+ semantic_name: "m".to_string(),+ llm_client: Some(Arc::new(BarrierClient {+ barrier: barrier.clone(),+ })),+ }]);+ // One shared algorithm driven by many concurrent requests.+ let algo = orch(targets);++ let mut handles = Vec::new();+ for _ in 0..N {+ let algo = algo.clone();+ handles.push(tokio::spawn(async move {+ algo.run(Context::default(), request())+ .await+ .map(|(_, response)| {+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default()+ })+ }));+ }++ for handle in handles {+ // The timeout turns a serialization deadlock into a failure, not a hang.+ let completion = tokio::time::timeout(Duration::from_secs(5), handle).await???;+ assert_eq!(completion, "m");+ }+ Ok(())+ }++ #[tokio::test]+ async fn offload_error_propagates_back_to_the_algorithm(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target offloads its call; we fulfill the promise with an+ // Err, which must flow back through `call_llm_target` into the algorithm and+ // out as an error step — not a response.+ let stream =+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut saw_error = false;+ while let Some(step) = stream.next().await {+ match step {+ Ok(Step::CallLlm(call)) => {+ call.respond(Err("upstream model call failed".into()))?;+ }+ Ok(Step::Decision(_)) => {}+ Ok(Step::ReturnToAgent(..)) => {+ return Err("expected the offload error to propagate, got a response".into());+ }+ Err(err) => {+ // The algorithm's `call_llm_target` saw the error via the promise.+ assert!(err.to_string().contains("upstream model call failed"));+ saw_error = true;+ }+ }+ }++ assert!(saw_error, "expected an error step");+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::time::Duration;+ use tokio::sync::{mpsc, Semaphore};++ // Mirrors `run`'s private `MAX_CONCURRENT_CALLS`. The algorithm fans out more+ // calls than the cap; only `CAP` should ever be in flight, the rest wait.+ const CAP: usize = 10;+ const TOTAL_CALLS: usize = CAP + 5;++ // Records peak concurrency and holds each call open until the gate is released,+ // so calls pile up and the cap can be observed.+ struct ProbeClient {+ current: Arc<AtomicUsize>,+ max: Arc<AtomicUsize>,+ entered: mpsc::UnboundedSender<()>,+ gate: Arc<Semaphore>,+ }++ #[async_trait]+ impl RoutedLlmClient for ProbeClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let now = self.current.fetch_add(1, Ordering::SeqCst) + 1;+ self.max.fetch_max(now, Ordering::SeqCst);+ let _ = self.entered.send(());+ // Block until the test opens the gate.+ let permit = self.gate.acquire().await.map_err(|e| e.to_string())?;+ drop(permit);+ self.current.fetch_sub(1, Ordering::SeqCst);+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ decision.selected_model().to_string(),+ )),+ metadata: None,+ })+ }+ }++ // Fans out `n` concurrent calls to one target.+ struct FanOut {+ n: usize,+ target: LlmTarget,+ }++ #[async_trait]+ impl Algorithm for FanOut {+ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ _request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let calls = (0..self.n).map(|i| {+ let driver = driver.clone();+ let ctx = ctx.clone();+ let target = self.target.clone();+ async move {+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {+ model: format!("m{i}"),+ });+ driver+ .call_llm_target(ctx, &target, request(), decision)+ .await+ }+ });+ futures::future::join_all(calls).await;+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),+ metadata: None,+ })+ }+ }++ let current = Arc::new(AtomicUsize::new(0));+ let max = Arc::new(AtomicUsize::new(0));+ let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();+ let gate = Arc::new(Semaphore::new(0)); // starts closed++ let client = Arc::new(ProbeClient {+ current: current.clone(),+ max: max.clone(),+ entered: entered_tx,+ gate: gate.clone(),+ }) as Arc<dyn RoutedLlmClient>;+ let target = LlmTarget {+ semantic_name: "m".to_string(),+ llm_client: Some(client),+ };++ let algo: Arc<dyn Algorithm> = Arc::new(FanOut {+ n: TOTAL_CALLS,+ target,+ });+ let handle = tokio::spawn(async move { algo.run(Context::default(), request()).await });++ // Exactly `CAP` calls should enter; each `recv` times out into an error if the+ // run dispatched fewer than the cap.+ for _ in 0..CAP {+ tokio::time::timeout(Duration::from_secs(5), entered_rx.recv())+ .await+ .map_err(|_| "timed out waiting for a call to enter")?;+ }+ assert_eq!(+ current.load(Ordering::SeqCst),+ CAP,+ "exactly the cap should be in flight"+ );+ // No further call enters while the cap is saturated — extra steps wait for capacity.+ assert!(+ tokio::time::timeout(Duration::from_millis(200), entered_rx.recv())+ .await+ .is_err(),+ "a call beyond the cap entered while {CAP} were already in flight"+ );++ // Release the gate; every call completes and the run finishes.+ gate.add_permits(TOTAL_CALLS);+ let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "done"+ );+ assert_eq!(+ max.load(Ordering::SeqCst),+ CAP,+ "peak in-flight should equal the cap"+ );+ Ok(())+ }+}diff --git a/crates/libsy/src/driver.rs b/crates/libsy/src/core/driver.rssimilarity index 98%rename from crates/libsy/src/driver.rsrename to crates/libsy/src/core/driver.rsindex b0bb0c770..093b6e306 100644--- a/crates/libsy/src/driver.rs+++ b/crates/libsy/src/core/driver.rs@@ -3,9 +3,10 @@//! # driver — a type-erased promise-over-a-stream request pump//!-//! [`TypeErasedDriver`] is the generic offload primitive; the crate root builds the-//! libsy-typed `Driver` on top of it. This module has no dependency on the rest of the-//! crate — the coupling is one-directional (`lib.rs` → `driver.rs`).+//! [`TypeErasedDriver`] is the generic offload primitive; its sibling+//! [`algorithm`](super::algorithm) builds the libsy-typed `Driver` on top of it. This+//! module has no dependency on the rest of the crate — the coupling is one-directional+//! (`core/algorithm.rs` → `core/driver.rs`).//!//! A [`TypeErasedDriver`] lets a *producer* (e.g. a routing algorithm) fulfill//! arbitrary requests by publishing promises onto a stream that a single *consumer*diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 2cf6ecf3b..2f8eebca8 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -54,947 +54,14 @@//!//! ## Algorithms//!-//! [`RandomAlgo`] provides uniform random routing in the core crate.+//! Concrete algorithms live in [`algorithms`]://!-//! [`LlmClassifierOrchAlgo`] classifies with one model, then routes to a-//! strong/weak model depending on the classifier's choice.--mod algorithms;-pub use algorithms::llm_class::{ClassifierDecision, ClassifierTier, LlmClassifierOrchAlgo};-pub use algorithms::noop::{NoopAlgo, NoopDecision};-pub use algorithms::rand::{RandomAlgo, RandomDecision};--mod driver;--use std::{error::Error, pin::Pin, sync::Arc};--use async_trait::async_trait;-use futures::{Stream, StreamExt};--/// The request/response protocol types, re-exported from [`switchyard_protocol`].-/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;-/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response-/// (a live [`LlmResponseStream`] or the terminal aggregate).-pub use switchyard_protocol::{- AggLlmResponse, Context, Decision, LlmRequest, LlmResponse, LlmResponseChunk,- LlmResponseStream, Metadata, Request, Response, RoutedLlmClient,-};--use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};--/// Shorthand for the crate's boxed, thread-safe error type.-type BoxErr = Box<dyn Error + Send + Sync>;--/// A boxed, `Send` stream of [`Step`]s — the output of-/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps-/// `Arc<dyn Algorithm>` object-safe.-pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;--/// Agentic-stack events fed to an algorithm out of band via-/// [`Algorithm::process_signals`] (e.g. tool results, budget updates).-///-/// A placeholder today; a stateful algorithm can begin consuming signals as the-/// enum grows without changing the orchestrator contract.-#[derive(Clone)]-pub struct Signals {}--/// A request paired with the routing [`Decision`] that produced it — the offload-/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.-///-/// The two model identifiers live in separate, unambiguous places: the model to-/// call is [`decision.selected_model()`](Decision::selected_model), while-/// `request.llm_request.model` is the *inbound* name the agent asked for (libsy-/// never overwrites it). A client maps `selected_model()` to the provider model-/// id it hits.-#[derive(Clone)]-pub struct RoutedRequest {- /// The request to serve; its `model` is the agent's original name.- pub request: Request,- /// The routing decision behind this call; `selected_model()` is the model to hit.- pub decision: Arc<dyn Decision>,- /// The client that serves this call by default, or `None` when the routed target- /// had no client. Rides along on the offloaded call so a host driving the stream- /// can serve it by default or override it with its own transport.- pub default_client: Option<Arc<dyn RoutedLlmClient>>,- /// The request's cross-cutting context, carried through the offload so whoever- /// serves the call (libsy's own `run`, or a host driving the stream) hands it to- /// [`RoutedLlmClient::call`].- pub ctx: Context,-}--/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].-///-/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the-/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it-/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and-/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's-/// [`Driver::call_llm`] on the other side.-pub struct CallLlmRequest {- inner: DriverRequest,- routed: RoutedRequest,-}--impl CallLlmRequest {- /// Wrap a driver request whose payload is a [`RoutedRequest`]. Caches an owned copy- /// so the accessors are plain field reads.- fn new(inner: DriverRequest) -> Self {- // The payload is always a `RoutedRequest` (set by `Driver::call_llm`); a- // mismatch would be a libsy bug, not a runtime condition.- let routed = match inner.request::<RoutedRequest>() {- Ok(routed) => routed.clone(),- Err(_) => unreachable!("CallLlmRequest payload is always a RoutedRequest"),- };- Self { inner, routed }- }-- /// The routed request the host should serve. Its- /// [`default_client`](RoutedRequest::default_client) serves the call by default,- /// and its `decision.selected_model()` names the model to hit.- pub fn get_routed(&self) -> &RoutedRequest {- &self.routed- }-- /// The model request to perform (the [`Request`] inside the routed request).- pub fn get_request(&self) -> &Request {- &self.get_routed().request- }-- /// The decision that led to this call — its `selected_model()` is the model to hit.- pub fn get_decision(&self) -> &dyn Decision {- self.get_routed().decision.as_ref()- }-- /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to- /// propagate a failed model call back to the algorithm. Consumes the promise: it- /// can only be fulfilled once.- pub fn respond(self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {- self.inner.respond::<Response>(result)- }-}--/// The offload channel handed to an algorithm's-/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls-/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and-/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the-/// request's [`Step`] stream and awaits the consumer's response. The step channel is-/// bounded, so the consumer paces the algorithm one step at a time.-#[derive(Clone)]-pub struct Driver {- driver: TypeErasedDriver,-}--impl Driver {- /// Build an empty driver with its step channel ready. Created per call by- /// [`run_stream`](Algorithm::run_stream).- pub(crate) fn new() -> Self {- Self {- driver: TypeErasedDriver::new(),- }- }-- /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the- /// consumer's [`Response`]. The call's context travels inside- /// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.- pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {- self.driver- .fulfill_request::<RoutedRequest, Response>(routed.ctx.clone(), routed)- .await- }-- /// Offload a call to `target`: pair `request` with `decision` and the target's- /// default client into a [`RoutedRequest`], then publish it (see- /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;- /// `decision.selected_model()` names the model to hit, and `request`'s- /// `model` is left untouched.- pub async fn call_llm_target(- &self,- ctx: Context,- target: &LlmTarget,- request: Request,- decision: Arc<dyn Decision>,- ) -> Result<Response, BoxErr> {- self.call_llm(RoutedRequest {- request,- decision,- default_client: target.llm_client.clone(),- ctx,- })- .await- }-- /// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.- pub async fn info(&self, ctx: Context, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {- self.driver.info(ctx, decision).await- }-- /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream- /// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)- /// when the algorithm finishes.- pub(crate) async fn finish(- &self,- ctx: Context,- result: Result<Response, BoxErr>,- ) -> Result<(), BoxErr> {- match result {- Ok(response) => self.driver.done(ctx, response).await,- Err(err) => self.driver.fail(ctx, err).await,- }- }-- /// Transform the raw driver stream into a stream of [`Step`]s. Internal: the- /// consumer stream is taken (once) by [`run_stream`](Algorithm::run_stream). A- /// payload that does not match the expected type for its step becomes an `Err` item.- pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {- self.driver.stream().map(|item| match item? {- DriverStep::Request(req) => Ok(Step::CallLlm(Box::new(CallLlmRequest::new(req)))),- DriverStep::Info(payload) => payload- .downcast::<Arc<dyn Decision>>()- .map(|decision| Step::Decision(*decision))- .map_err(|_| "driver: info payload was not a Decision".into()),- DriverStep::Done(payload) => payload- .downcast::<Response>()- .map(Step::ReturnToAgent)- .map_err(|_| "driver: done payload was not a Response".into()),- })- }-}--impl Default for Driver {- fn default() -> Self {- Self::new()- }-}--/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].-pub enum Step {- /// The algorithm needs this model call performed. The host serves it (optionally- /// via [`RoutedRequest::default_client`]) and fulfills it with- /// [`CallLlmRequest::respond`]. Boxed: it is by far the largest variant.- CallLlm(Box<CallLlmRequest>),- /// A routing decision the algorithm made, published via [`Driver::info`] as it- /// happens (rather than collected into a trace returned at the end).- Decision(Arc<dyn Decision>),- /// The algorithm finished with its final response — the last step of a run.- ReturnToAgent(Box<Response>),-}--/// A named routing target: a `semantic_name` an algorithm routes by, and an optional-/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to-/// [`Driver::call_llm_target`]; the client rides along as-/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.-#[derive(Clone)]-pub struct LlmTarget {- /// The routing label an algorithm selects this target by — a logical tier like- /// `"strong"`, or the model id when they coincide. Mapping it to a provider model- /// id is the client's concern, never the algorithm's.- pub semantic_name: String,- /// The client that serves this target's calls by default, or `None` (then the- /// stream consumer must serve them).- pub llm_client: Option<Arc<dyn RoutedLlmClient>>,-}--/// The set of targets an algorithm may route among. An algorithm is constructed-/// with one and picks targets by position ([`targets`](Self::targets)) or by name-/// ([`get_target`](Self::get_target)).-#[derive(Clone)]-pub struct LlmTargetSet {- targets: Vec<LlmTarget>,-}--impl LlmTargetSet {- /// Build a target set from a list of targets.- pub fn new(targets: Vec<LlmTarget>) -> Self {- Self { targets }- }-- /// All targets in the set — e.g. for an algorithm to select among.- pub fn targets(&self) -> &[LlmTarget] {- &self.targets- }-- /// Look up a target by name; errors if no target has that name.- pub fn get_target(&self, name: &str) -> Result<LlmTarget, Box<dyn Error + Send + Sync>> {- self.targets- .iter()- .find(|t| t.semantic_name == name)- .cloned()- .ok_or(format!("Target {} not found", name).into())- }-}--/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);-/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)-/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).-///-/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across-/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms-/// (the reference routers) get this for free; a stateful one uses interior mutability-/// over just its own state.-#[async_trait]-pub trait Algorithm: Send + Sync + 'static {- /// Run one request to completion: make model calls with [`Driver::call_llm_target`],- /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].- /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)- /// drive it. `ctx` carries cross-cutting request state (empty today).- async fn create_run_task(- self: Arc<Self>,- ctx: Context,- driver: Driver,- request: Request,- ) -> Result<Response, Box<dyn Error + Send + Sync>>;-- /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The- /// reference algorithms ignore signals; a stateful algorithm updates its own- /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.- #[allow(unused_variables)]- async fn process_signals(- self: Arc<Self>,- signals: Signals,- ) -> Result<(), Box<dyn Error + Send + Sync>> {- Ok(())- }-- /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its- /// own task; drive the stream: serve each [`Step::CallLlm`] (via its- /// [`default_client`](RoutedRequest::default_client) or your own transport) and read- /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is- /// bounded, so pulling paces the algorithm; each call is independent, so many run- /// concurrently.- fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {- // This call's own driver: take its consumer stream, hand a producer-side clone to- // the algorithm task, and keep one to emit the terminal step. The task blocks- // publishing a step until the consumer pulls the previous one.- let driver = Driver::new();- let stream = driver.stream();- tokio::spawn(async move {- let outcome = self- .create_run_task(ctx.clone(), driver.clone(), request)- .await;- let _ = driver.finish(ctx, outcome).await;- });- Box::pin(stream)- }-- /// Run one request to completion, serving each offloaded call with its- /// [`RoutedRequest::default_client`], and return the decision trace plus the final- /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,- /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model- /// clients and the host wants the answer (and the decisions behind it); drive- /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned- /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]- /// or a live [`LlmResponse::Stream`]; the caller decides whether to- /// [`aggregate`](LlmResponse::aggregate) it or forward its chunks. Errors if a routed- /// target has no client to serve its call, or the algorithm fails.- async fn run(- self: Arc<Self>,- ctx: Context,- request: Request,- ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {- // Serve up to this many offloaded calls concurrently, so an algorithm that- // fans out (e.g. an ensemble) isn't serialized on the client.- const MAX_CONCURRENT_CALLS: usize = 10;-- // Serve one offloaded call with its target's default client. A failed *model*- // call is forwarded to the algorithm via `respond`; this errors only on an- // infrastructure failure (no default client, or the promise was dropped).- async fn serve(call: CallLlmRequest) -> Result<(), Box<dyn Error + Send + Sync>> {- let routed = call.get_routed().clone();- let client = routed.default_client.clone().ok_or_else(|| {- format!(- "run: target '{}' has no client to serve the call",- routed.decision.selected_model()- )- })?;- call.respond(- client- .call(routed.ctx, routed.request, routed.decision)- .await,- )- }-- let stream = self.run_stream(ctx, request);- tokio::pin!(stream);-- let mut trace: Vec<Arc<dyn Decision>> = Vec::new();- let mut in_flight = futures::stream::FuturesUnordered::new();- let mut final_response: Option<Response> = None;- let mut stream_open = true;-- while stream_open || !in_flight.is_empty() {- tokio::select! {- // Surface a failed serve as soon as it completes.- Some(result) = in_flight.next(), if !in_flight.is_empty() => result?,- // Pull the next step, unless the stream ended or we're at the cap.- step = stream.next(), if stream_open && in_flight.len() < MAX_CONCURRENT_CALLS => {- match step {- None => stream_open = false,- Some(item) => match item? {- Step::CallLlm(call) => in_flight.push(serve(*call)),- Step::Decision(decision) => trace.push(decision),- Step::ReturnToAgent(response) => {- final_response = Some(*response);- stream_open = false;- }- },- }- }- }- }-- // Return whatever the algorithm produced — buffered or streamed. The caller- // decides whether to aggregate the response or forward its chunks.- final_response- .map(|response| (trace, response))- .ok_or_else(|| "run: stream ended without a final response".into())- }-}--#[cfg(test)]-mod tests {- use super::*;- use futures::StreamExt;- use switchyard_protocol::{completion_text, text_request, text_response};-- /// Mock client that echoes back the target name it was called with.- struct EchoClient;-- #[async_trait]- impl RoutedLlmClient for EchoClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- // Echo back the model the algorithm routed to (the decision's selection).- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-- /// Trivial decision + algo used only to exercise the orchestrator: calls the- /// first target and returns its response with a one-item trace.- struct TestDecision {- model: String,- }-- impl Decision for TestDecision {- fn selected_model(&self) -> &str {- &self.model- }- fn reasoning(&self) -> Option<&str> {- None- }- fn as_any(&self) -> &dyn std::any::Any {- self- }- }-- struct TestAlgo {- target_set: LlmTargetSet,- }-- #[async_trait]- impl Algorithm for TestAlgo {- async fn create_run_task(- self: Arc<Self>,- ctx: Context,- driver: Driver,- request: Request,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- let target = self- .target_set- .targets()- .first()- .ok_or("no targets")?- .clone();- let decision: Arc<dyn Decision> = Arc::new(TestDecision {- model: target.semantic_name.clone(),- });- driver.info(ctx.clone(), decision.clone()).await?;- driver- .call_llm_target(ctx, &target, request, decision)- .await- }- }-- /// Build a shared `TestAlgo` over the given target set.- fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {- Arc::new(TestAlgo { target_set })- }-- fn request() -> Request {- Request {- llm_request: text_request(Some("auto".to_string()), "hi".to_string()),- raw_request: None,- metadata: None,- }- }-- /// `(name, has_client)` — `has_client: false` builds a target with no default client.- fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {- let targets = names- .iter()- .map(|(name, has_client)| LlmTarget {- semantic_name: name.to_string(),- llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn RoutedLlmClient>),- })- .collect();- LlmTargetSet::new(targets)- }-- /// Client that serves a call as a token stream — its `call` returns- /// [`LlmResponse::Stream`] replaying `chunks` in order (as `Ok` items).- struct StreamingClient {- chunks: Vec<LlmResponseChunk>,- }-- #[async_trait]- impl RoutedLlmClient for StreamingClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- _decision: Arc<dyn Decision>,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();- Ok(Response {- llm_response: LlmResponse::Stream(stream),- metadata: None,- })- }- }-- /// Build a single-target algo whose one target streams `chunks`.- fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> Arc<dyn Algorithm> {- let target = LlmTarget {- semantic_name: "stream/model".to_string(),- llm_client: Some(Arc::new(StreamingClient { chunks }) as Arc<dyn RoutedLlmClient>),- };- orch(LlmTargetSet::new(vec![target]))- }-- #[tokio::test]- async fn run_returns_a_streamed_response_the_caller_aggregates(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,- // and `run` returns the live stream untouched for the caller to fold.- let orch = streaming_orch(vec![- LlmResponseChunk::MessageStart {- id: Some("m1".to_string()),- model: Some("stream/model".to_string()),- },- LlmResponseChunk::TextDelta {- index: 0,- text: "hel".to_string(),- },- LlmResponseChunk::TextDelta {- index: 0,- text: "lo".to_string(),- },- LlmResponseChunk::MessageStop {- reason: Some("stop".to_string()),- },- ]);- let (trace, response) = orch.run(Context::default(), request()).await?;- // `run` handed back the live stream; the caller folds it to a buffered aggregate.- let agg = response.llm_response.into_agg().await?;- assert_eq!(completion_text(&agg), "hello");- assert_eq!(agg.model.as_deref(), Some("stream/model"));- assert_eq!(trace.len(), 1);- Ok(())- }-- #[tokio::test]- async fn aggregating_a_streamed_response_propagates_a_mid_stream_error(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // `run` succeeds and returns the stream; the in-band `Error` chunk surfaces only- // when the caller aggregates it.- let orch = streaming_orch(vec![- LlmResponseChunk::TextDelta {- index: 0,- text: "partial".to_string(),- },- LlmResponseChunk::Error {- message: "upstream exploded".to_string(),- },- ]);- let (_, response) = orch.run(Context::default(), request()).await?;- match response.llm_response.into_agg().await {- Ok(_) => Err("expected a mid-stream error, got an aggregate".into()),- Err(err) => {- assert!(err.to_string().contains("upstream exploded"));- Ok(())- }- }- }-- #[tokio::test]- async fn run_offloads_via_promise_then_returns_to_agent(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // A client-less target -> its call is offloaded via a promise the- // orchestrator surfaces as a `CallLlm` step for us to fulfill.- let stream =- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());- tokio::pin!(stream);-- let mut saw_call = false;- let mut final_completion = None;- while let Some(step) = stream.next().await {- match step? {- Step::CallLlm(call) => {- saw_call = true;- // The decision rode along with the promise.- assert_eq!(call.get_decision().selected_model(), "offload/model");- // Fulfilling the promise is the "real" model call the caller makes.- call.respond(Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- "fulfilled".to_string(),- )),- metadata: None,- }))?;- }- Step::Decision(decision) => {- assert_eq!(decision.selected_model(), "offload/model");- }- Step::ReturnToAgent(response) => {- final_completion = Some(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- );- }- }- }-- assert!(saw_call, "expected a CallLlm step before ReturnToAgent");- assert_eq!(- final_completion.ok_or("no ReturnToAgent step")?,- "fulfilled"- );- Ok(())- }-- #[tokio::test]- async fn client_backed_target_offloads_with_a_default_client(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // Every call now offloads to the stream; a client-backed target rides its- // client along as `default_client` so the consumer can serve it by default.- let stream =- orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());- tokio::pin!(stream);-- let mut final_completion = None;- while let Some(step) = stream.next().await {- match step? {- Step::CallLlm(call) => {- let routed = call.get_routed().clone();- let client = routed- .default_client- .clone()- .ok_or("expected a default client")?;- let result = client- .call(routed.ctx, routed.request, routed.decision)- .await;- call.respond(result)?;- }- Step::Decision(_) => {}- Step::ReturnToAgent(response) => {- final_completion = Some(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- );- }- }- }-- // EchoClient echoes the model name back as the completion.- assert_eq!(final_completion.ok_or("no ReturnToAgent")?, "direct/model");- Ok(())- }-- #[tokio::test]- async fn run_returns_the_response_when_all_targets_have_clients(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // Every target has a client, so run serves every call via the- // default client and returns the trace + final response.- let (trace, response) = orch(target_set(&[("direct/model", true)]))- .run(Context::default(), request())- .await?;- // TestAlgo calls the first target; EchoClient echoes its name.- assert_eq!(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- "direct/model"- );- assert_eq!(trace[0].selected_model(), "direct/model");- Ok(())- }-- #[tokio::test]- async fn run_errors_when_a_target_lacks_a_client() -> Result<(), Box<dyn Error + Send + Sync>> {- // A client-less target has no default client to serve its offloaded call, so- // driving it to completion errors.- assert!(orch(target_set(&[("offload/model", false)]))- .run(Context::default(), request())- .await- .is_err());- Ok(())- }-- #[tokio::test(flavor = "multi_thread", worker_threads = 12)]- async fn requests_are_processed_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {- use std::time::Duration;- use tokio::sync::Barrier;-- const N: usize = 12;-- // A client that blocks until all N concurrent calls have arrived. If- // requests were serialized (one algorithm behind a `Mutex`), only one- // call could be in flight, the barrier would never reach N, and the test- // would time out. It passes only because the shared algorithm is driven- // concurrently across requests.- struct BarrierClient {- barrier: Arc<Barrier>,- }-- #[async_trait]- impl RoutedLlmClient for BarrierClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- self.barrier.wait().await;- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-- let barrier = Arc::new(Barrier::new(N));- let targets = LlmTargetSet::new(vec![LlmTarget {- semantic_name: "m".to_string(),- llm_client: Some(Arc::new(BarrierClient {- barrier: barrier.clone(),- })),- }]);- // One shared algorithm driven by many concurrent requests.- let algo = orch(targets);-- let mut handles = Vec::new();- for _ in 0..N {- let algo = algo.clone();- handles.push(tokio::spawn(async move {- algo.run(Context::default(), request())- .await- .map(|(_, response)| {- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default()- })- }));- }-- for handle in handles {- // The timeout turns a serialization deadlock into a failure, not a hang.- let completion = tokio::time::timeout(Duration::from_secs(5), handle).await???;- assert_eq!(completion, "m");- }- Ok(())- }-- #[tokio::test]- async fn offload_error_propagates_back_to_the_algorithm(- ) -> Result<(), Box<dyn Error + Send + Sync>> {- // A client-less target offloads its call; we fulfill the promise with an- // Err, which must flow back through `call_llm_target` into the algorithm and- // out as an error step — not a response.- let stream =- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());- tokio::pin!(stream);-- let mut saw_error = false;- while let Some(step) = stream.next().await {- match step {- Ok(Step::CallLlm(call)) => {- call.respond(Err("upstream model call failed".into()))?;- }- Ok(Step::Decision(_)) => {}- Ok(Step::ReturnToAgent(..)) => {- return Err("expected the offload error to propagate, got a response".into());- }- Err(err) => {- // The algorithm's `call_llm_target` saw the error via the promise.- assert!(err.to_string().contains("upstream model call failed"));- saw_error = true;- }- }- }-- assert!(saw_error, "expected an error step");- Ok(())- }-- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]- async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {- use std::sync::atomic::{AtomicUsize, Ordering};- use std::time::Duration;- use tokio::sync::{mpsc, Semaphore};-- // Mirrors `run`'s private `MAX_CONCURRENT_CALLS`. The algorithm fans out more- // calls than the cap; only `CAP` should ever be in flight, the rest wait.- const CAP: usize = 10;- const TOTAL_CALLS: usize = CAP + 5;-- // Records peak concurrency and holds each call open until the gate is released,- // so calls pile up and the cap can be observed.- struct ProbeClient {- current: Arc<AtomicUsize>,- max: Arc<AtomicUsize>,- entered: mpsc::UnboundedSender<()>,- gate: Arc<Semaphore>,- }-- #[async_trait]- impl RoutedLlmClient for ProbeClient {- async fn call(- &self,- _ctx: Context,- _request: Request,- decision: Arc<dyn Decision>,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- let now = self.current.fetch_add(1, Ordering::SeqCst) + 1;- self.max.fetch_max(now, Ordering::SeqCst);- let _ = self.entered.send(());- // Block until the test opens the gate.- let permit = self.gate.acquire().await.map_err(|e| e.to_string())?;- drop(permit);- self.current.fetch_sub(1, Ordering::SeqCst);- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })- }- }-- // Fans out `n` concurrent calls to one target.- struct FanOut {- n: usize,- target: LlmTarget,- }-- #[async_trait]- impl Algorithm for FanOut {- async fn create_run_task(- self: Arc<Self>,- ctx: Context,- driver: Driver,- _request: Request,- ) -> Result<Response, Box<dyn Error + Send + Sync>> {- let calls = (0..self.n).map(|i| {- let driver = driver.clone();- let ctx = ctx.clone();- let target = self.target.clone();- async move {- let decision: Arc<dyn Decision> = Arc::new(TestDecision {- model: format!("m{i}"),- });- driver- .call_llm_target(ctx, &target, request(), decision)- .await- }- });- futures::future::join_all(calls).await;- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),- metadata: None,- })- }- }-- let current = Arc::new(AtomicUsize::new(0));- let max = Arc::new(AtomicUsize::new(0));- let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();- let gate = Arc::new(Semaphore::new(0)); // starts closed-- let client = Arc::new(ProbeClient {- current: current.clone(),- max: max.clone(),- entered: entered_tx,- gate: gate.clone(),- }) as Arc<dyn RoutedLlmClient>;- let target = LlmTarget {- semantic_name: "m".to_string(),- llm_client: Some(client),- };-- let algo: Arc<dyn Algorithm> = Arc::new(FanOut {- n: TOTAL_CALLS,- target,- });- let handle = tokio::spawn(async move { algo.run(Context::default(), request()).await });+//! [`algorithms::Random`](crate::algorithms::Random) provides uniform random routing.+//!+//! [`algorithms::LlmClassifier`](crate::algorithms::LlmClassifier) classifies+//! with one model, then routes to a strong/weak model depending on the classifier's choice.- // Exactly `CAP` calls should enter; each `recv` times out into an error if the- // run dispatched fewer than the cap.- for _ in 0..CAP {- tokio::time::timeout(Duration::from_secs(5), entered_rx.recv())- .await- .map_err(|_| "timed out waiting for a call to enter")?;- }- assert_eq!(- current.load(Ordering::SeqCst),- CAP,- "exactly the cap should be in flight"- );- // No further call enters while the cap is saturated — extra steps wait for capacity.- assert!(- tokio::time::timeout(Duration::from_millis(200), entered_rx.recv())- .await- .is_err(),- "a call beyond the cap entered while {CAP} were already in flight"- );+mod core;+pub use core::*;- // Release the gate; every call completes and the run finishes.- gate.add_permits(TOTAL_CALLS);- let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;- assert_eq!(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- "done"- );- assert_eq!(- max.load(Ordering::SeqCst),- CAP,- "peak in-flight should equal the cap"- );- Ok(())- }-}+pub mod algorithms;diff --git a/crates/libsy-protocol/Cargo.toml b/crates/protocol/Cargo.tomlsimilarity index 100%rename from crates/libsy-protocol/Cargo.tomlrename to crates/protocol/Cargo.tomldiff --git a/crates/libsy-protocol/LICENSE b/crates/protocol/LICENSEsimilarity index 100%rename from crates/libsy-protocol/LICENSErename to crates/protocol/LICENSEdiff --git a/crates/libsy-protocol/NOTICE b/crates/protocol/NOTICEsimilarity index 100%rename from crates/libsy-protocol/NOTICErename to crates/protocol/NOTICEdiff --git a/crates/libsy-protocol/README.md b/crates/protocol/README.mdsimilarity index 100%rename from crates/libsy-protocol/README.mdrename to crates/protocol/README.mddiff --git a/crates/libsy-protocol/src/client.rs b/crates/protocol/src/client.rssimilarity index 100%rename from crates/libsy-protocol/src/client.rsrename to crates/protocol/src/client.rsdiff --git a/crates/libsy-protocol/src/envelope.rs b/crates/protocol/src/envelope.rssimilarity index 90%rename from crates/libsy-protocol/src/envelope.rsrename to crates/protocol/src/envelope.rsindex 9906bd866..1781d3637 100644--- a/crates/libsy-protocol/src/envelope.rs+++ b/crates/protocol/src/envelope.rs@@ -15,6 +15,14 @@ pub struct Context {pub values: HashMap<String, String>,}+/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget+/// updates) — in libsy, via `Algorithm::process_signals`.+///+/// A placeholder today; a stateful algorithm can begin consuming signals as the enum+/// grows without changing the orchestrator contract.+#[derive(Clone)]+pub struct Signals {}+/// Correlation and routing metadata attached to a request or response.////// All fields are optional; algorithms and observers use whichever are presentdiff --git a/crates/libsy-protocol/src/format.rs b/crates/protocol/src/format.rssimilarity index 100%rename from crates/libsy-protocol/src/format.rsrename to crates/protocol/src/format.rsdiff --git a/crates/libsy-protocol/src/lib.rs b/crates/protocol/src/lib.rssimilarity index 100%rename from crates/libsy-protocol/src/lib.rsrename to crates/protocol/src/lib.rsdiff --git a/crates/libsy-protocol/src/llm.rs b/crates/protocol/src/llm.rssimilarity index 100%rename from crates/libsy-protocol/src/llm.rsrename to crates/protocol/src/llm.rsdiff --git a/crates/libsy-protocol/src/stream.rs b/crates/protocol/src/stream.rssimilarity index 100%rename from crates/libsy-protocol/src/stream.rsrename to crates/protocol/src/stream.rsdiff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.tomlindex 5659430c1..074625a67 100644--- a/crates/switchyard-py/Cargo.toml+++ b/crates/switchyard-py/Cargo.toml@@ -18,7 +18,7 @@ crate-type = ["cdylib", "rlib"][dependencies]async-trait = "0.1"futures-util = "0.3"-libsy = { path = "../libsy" }+switchyard-libsy = { path = "../libsy" }pyo3 = { version = "0.28.3", features = ["abi3-py312", "extension-module"] }pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }pythonize = "0.28.0"diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex a7e62ef32..02ce29037 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -7,13 +7,14 @@ use std::error::Error;use std::sync::Arc;use async_trait::async_trait;-use libsy::{- AggLlmResponse, Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, NoopAlgo,- RandomAlgo, Request, Response, RoutedLlmClient,-};use pyo3::exceptions::{PyTypeError, PyValueError};use pyo3::prelude::*;use serde_json::{json, Value};+use switchyard_libsy::algorithms::{Noop, Random};+use switchyard_libsy::{+ AggLlmResponse, Algorithm, Context, Decision, LlmResponse, LlmTarget, LlmTargetSet, Request,+ Response, RoutedLlmClient,+};use crate::errors::py_libsy_error;use crate::py_serde::{from_python, to_python};@@ -148,7 +149,7 @@ impl PyAlgorithm {/// Construct the no-op reference algorithm.#[pyfunction(name = "noop")]fn noop_algorithm() -> PyAlgorithm {- PyAlgorithm::new(Arc::new(NoopAlgo {}))+ PyAlgorithm::new(Arc::new(Noop {}))}/// Construct uniform random routing over targets with Python clients.@@ -161,9 +162,9 @@ fn random_algorithm(py: Python<'_>, targets: Vec<Py<PyLlmTarget>>) -> PyResult<P.iter().map(|target| Ok(target.bind(py).try_borrow()?.clone_core(py))).collect::<PyResult<Vec<_>>>()?;- Ok(PyAlgorithm::new(Arc::new(RandomAlgo::new(- LlmTargetSet::new(targets),- ))))+ Ok(PyAlgorithm::new(Arc::new(Random::new(LlmTargetSet::new(+ targets,+ )))))}fn boxed_python_error(error: PyErr) -> BoxError {diff --git a/crates/switchyard-translation/Cargo.toml b/crates/switchyard-translation/Cargo.tomlindex 9e55e7e9c..7f15b75da 100644--- a/crates/switchyard-translation/Cargo.toml+++ b/crates/switchyard-translation/Cargo.toml@@ -19,7 +19,7 @@ publish = ["crates-io"][dependencies]serde = { version = "1", features = ["derive"] }serde_json = "1"-switchyard-protocol = { path = "../libsy-protocol", version = "0.1.0" }+switchyard-protocol = { path = "../protocol", version = "0.1.0" }thiserror = "2"futures = "0.3"async-stream = "0.3"