Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): core interfaces and examples (#17)
Greg Clarkbfabc3bf147d
diff --git a/Cargo.lock b/Cargo.lockindex 402d83896..e73214a56 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -416,6 +416,21 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"+[[package]]+name = "futures"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"+dependencies = [+ "futures-channel",+ "futures-core",+ "futures-executor",+ "futures-io",+ "futures-sink",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-channel"version = "0.3.32"@@ -423,6 +438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"dependencies = ["futures-core",+ "futures-sink",][[package]]@@ -431,6 +447,17 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"+[[package]]+name = "futures-executor"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-io"version = "0.3.32"@@ -466,6 +493,7 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"dependencies = [+ "futures-channel","futures-core","futures-io","futures-macro",@@ -821,6 +849,29 @@ version = "0.2.186"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"+[[package]]+name = "libsy"+version = "0.1.0"+dependencies = [+ "async-trait",+ "futures",+ "serde_json",+ "tokio",+ "tokio-stream",+]++[[package]]+name = "libsy-examples"+version = "0.1.0"+dependencies = [+ "async-trait",+ "futures",+ "libsy",+ "rand 0.8.6",+ "tokio",+ "tokio-stream",+]+[[package]]name = "libyaml-rs"version = "0.3.0"@@ -1728,6 +1779,7 @@ dependencies = ["bytes","libc","mio",+ "parking_lot","pin-project-lite","signal-hook-registry","socket2",@@ -1756,6 +1808,17 @@ dependencies = ["tokio",]+[[package]]+name = "tokio-stream"+version = "0.1.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"+dependencies = [+ "futures-core",+ "pin-project-lite",+ "tokio",+]+[[package]]name = "tokio-util"version = "0.7.18"diff --git a/Cargo.toml b/Cargo.tomlindex 7c7c18eb4..3295507e0 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -4,6 +4,8 @@[workspace]resolver = "2"members = [+ "crates/libsy",+ "crates/libsy-examples","crates/switchyard-components","crates/switchyard-components-v2","crates/switchyard-components-v2-macros",diff --git a/crates/libsy-examples/Cargo.toml b/crates/libsy-examples/Cargo.tomlnew file mode 100644index 000000000..082bec560--- /dev/null+++ b/crates/libsy-examples/Cargo.toml@@ -0,0 +1,21 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "libsy-examples"+version = "0.1.0"+description = "Reference algorithms and runnable agents built on libsy"+publish = false+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true++[dependencies]+libsy = { path = "../libsy" }+async-trait = "0.1"+rand = "0.8"+futures = "0.3"+tokio = { version = "1", features = ["full"] }+tokio-stream = "0.1"diff --git a/crates/libsy-examples/examples/research_agent.rs b/crates/libsy-examples/examples/research_agent.rsnew file mode 100644index 000000000..8038ca383--- /dev/null+++ b/crates/libsy-examples/examples/research_agent.rs@@ -0,0 +1,106 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Minimal research agent using the [`Algorithm::run`] convenience.+//!+//! Every target owns an `LlmClient`, so the agent runs each request to completion with+//! [`Algorithm::run`]: it serves each offloaded call with the routed+//! target's `default_client` and returns the final response — no stream to drive. The+//! multi-step routing (classify -> route) happens inside the classifier algorithm; the+//! agent never sees it. To drive the step stream yourself instead, use+//! `Algorithm::run_stream`. Run with:+//! cargo run -p libsy --example research_agent++use std::error::Error;+use std::sync::Arc;++use async_trait::async_trait;+use libsy::{+ Algorithm, Context, LlmClient, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet, Request,+ Response, RoutedRequest,+};+use libsy_examples::llm_class::LlmClassifierOrchAlgo;++const CLASSIFIER: &str = "classifier/model";+const STRONG: &str = "strong/model";+const WEAK: &str = "weak/model";++/// Stub transport. Real integrators implement `LlmClient` over their own HTTP.+struct StubClient;++#[async_trait]+impl LlmClient for StubClient {+ async fn call(&self, routed: RoutedRequest) -> Result<Response, Box<dyn Error + Send + Sync>> {+ // The model to call is the routed decision's selection, not the inbound name.+ let model = routed.decision.selected_model().to_string();+ println!(" -> model call: {model}");+ // The classifier returns a score; other models return an answer.+ let completion = if model == CLASSIFIER {+ "0.9".to_string()+ } else {+ format!("answer from {model}")+ };+ Ok(Response {+ llm_response: LlmResponse {+ completion,+ raw_response: None,+ },+ metadata: None,+ })+ }+}++fn targets() -> LlmTargetSet {+ let client = Arc::new(StubClient) as Arc<dyn LlmClient>;+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ LlmTargetSet::new(vec![target(CLASSIFIER), target(STRONG), target(WEAK)])+}++struct ResearchAgent {+ algo: Arc<dyn Algorithm>,+}++impl ResearchAgent {+ /// Trivial plan: one lookup per question (stub).+ fn plan(&self, question: &str) -> Vec<String> {+ vec![format!("look up: {question}")]+ }++ async fn run(&self, question: &str) -> Result<String, Box<dyn Error + Send + Sync>> {+ let mut notes = Vec::new();+ for step in self.plan(question) {+ let request = Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: step,+ },+ raw_request: None,+ metadata: None,+ };++ let (_trace, response) = self.algo.clone().run(Context::default(), request).await?;+ notes.push(response.llm_response.completion);+ }+ Ok(notes.join("\n"))+ }+}++#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {+ // Configure routing once: an LLM classifier over three named targets. Swapping+ // in `RandomOrchAlgo` needs no change to the agent.+ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(+ CLASSIFIER,+ STRONG,+ WEAK,+ 0.5,+ targets(),+ ));++ let agent = ResearchAgent { algo };+ println!("{}", agent.run("what is switchyard?").await?);+ Ok(())+}diff --git a/crates/libsy-examples/examples/research_agent_core.rs b/crates/libsy-examples/examples/research_agent_core.rsnew file mode 100644index 000000000..bdb05ae8c--- /dev/null+++ b/crates/libsy-examples/examples/research_agent_core.rs@@ -0,0 +1,118 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Research agent driving the raw `run` stream with **client-less** targets.+//!+//! With no client, every `driver.call_llm_target` is offloaded as a promise the orchestrator+//! surfaces as a `CallLlm` step. The agent makes the "real" model call itself and+//! fulfills the promise — this is the offload/streaming path ("ask, don't call").+//! The classifier's two steps show up as two `model call:` lines. Run with:+//! cargo run -p libsy --example research_agent_core++use std::error::Error;+use std::sync::Arc;++use libsy::{+ Algorithm, Context, Decision, LlmRequest, LlmResponse, LlmTarget, LlmTargetSet, Request,+ Response, Step,+};+use libsy_examples::llm_class::LlmClassifierOrchAlgo;+use tokio_stream::StreamExt;++const CLASSIFIER: &str = "classifier/model";+const STRONG: &str = "strong/model";+const WEAK: &str = "weak/model";++/// The "real" model call the agent makes to fulfill a promise. The core never+/// makes the call itself — it hands back a request and waits for the response.+/// The model to call is the routing decision's selection, read off the promise.+async fn call_model(model: &str) -> Response {+ println!(" -> model call: {model}");+ let completion = if model == CLASSIFIER {+ "0.9".to_string()+ } else {+ format!("answer from {model}")+ };+ Response {+ llm_response: LlmResponse {+ completion,+ raw_response: None,+ },+ metadata: None,+ }+}++fn targets() -> LlmTargetSet {+ // Client-less targets -> every call is offloaded via a promise.+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: None,+ };+ LlmTargetSet::new(vec![target(CLASSIFIER), target(STRONG), target(WEAK)])+}++struct ResearchAgent {+ algo: Arc<dyn Algorithm>,+}++impl ResearchAgent {+ /// Trivial plan: one lookup per question (stub).+ fn plan(&self, question: &str) -> Vec<String> {+ vec![format!("look up: {question}")]+ }++ async fn run(&mut self, question: &str) -> Result<String, Box<dyn Error + Send + Sync>> {+ let mut notes = Vec::new();+ for step in self.plan(question) {+ let request = Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: step,+ },+ raw_request: None,+ metadata: None,+ };+ let stream = self.algo.clone().run_stream(Context::default(), request);+ tokio::pin!(stream);+ while let Some(update) = stream.next().await {+ match update? {+ Step::CallLlm(call) => {+ // Perform the model call the algorithm asked for, then fulfill.+ let response = call_model(call.get_decision()?.selected_model()).await;+ call.respond(Ok(response))?;+ }+ // Decisions stream in as the algorithm makes them.+ Step::Decision(decision) => print_decision(decision.as_ref()),+ Step::ReturnToAgent(response) => {+ notes.push(response.llm_response.completion);+ }+ }+ }+ }+ Ok(notes.join("\n"))+ }+}++/// Print one decision the algorithm recorded — uniform access via the trait.+fn print_decision(decision: &dyn Decision) {+ println!(+ " decision: {} ({})",+ decision.selected_model(),+ decision.reasoning().unwrap_or_default()+ );+}++#[tokio::main(flavor = "current_thread")]+async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {+ let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(+ CLASSIFIER,+ STRONG,+ WEAK,+ 0.5,+ targets(),+ ));++ let mut agent = ResearchAgent { algo };+ println!("{}", agent.run("what is switchyard?").await?);+ Ok(())+}diff --git a/crates/libsy-examples/src/ensemble.rs b/crates/libsy-examples/src/ensemble.rsnew file mode 100644index 000000000..4f4a9fc9d--- /dev/null+++ b/crates/libsy-examples/src/ensemble.rs@@ -0,0 +1,822 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Ensemble router built on the [`Algorithm`] interfaces.+//!+//! Each request is fanned out to a set of candidate models concurrently; a judge+//! model (e.g. Haiku) then picks the best response, which is returned to the+//! agent. The algorithm tallies which candidate the judge preferred across+//! requests and, after `exploration_turns` ensemble turns, commits to the+//! winningest model — every subsequent request routes straight to that one model+//! with no fan-out and no judge call.+//!+//! Unlike the reference routers, this algorithm is **stateful**: the win tally,+//! turn counter, and committed choice live behind a [`std::sync::Mutex`] so one+//! shared `&self` can serve a session's requests concurrently (see the+//! `Algorithm` docs). In a proxy setup one `EnsembleOrchAlgo` is created per session,+//! so this state is per-session.++use std::collections::BTreeMap;+use std::error::Error;+use std::sync::{Arc, Mutex};++use async_trait::async_trait;++use libsy::{+ Algorithm, Context, Decision, Driver, LlmRequest, LlmTargetSet, Request, Response, Signals,+};++/// Which step of the ensemble flow produced a decision.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum EnsemblePhase {+ /// A fan-out call to one candidate model during an exploration turn.+ Candidate,+ /// The judge call that scored the candidate responses.+ Judge,+ /// The candidate the judge selected on an exploration turn.+ Winner,+ /// The single model the algorithm committed to after exploration.+ Committed,+}++impl EnsemblePhase {+ /// Stable string form of the phase, used in decision reasoning.+ pub fn as_str(self) -> &'static str {+ match self {+ EnsemblePhase::Candidate => "candidate",+ EnsemblePhase::Judge => "judge",+ EnsemblePhase::Winner => "winner",+ EnsemblePhase::Committed => "committed",+ }+ }+}++/// Decision produced at each step of the ensemble flow.+pub struct EnsembleDecision {+ /// The model this step concerns (a candidate, the judge, the winner, or the+ /// committed model).+ pub selected_model: String,+ /// Human-readable explanation of the step.+ pub reasoning: String,+ /// Which step of the ensemble flow produced this decision.+ pub phase: EnsemblePhase,+}++impl Decision for EnsembleDecision {+ fn selected_model(&self) -> &str {+ &self.selected_model+ }+ fn reasoning(&self) -> Option<&str> {+ Some(&self.reasoning)+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+}++/// Mutable per-session state, guarded by a [`Mutex`] so `&self` can be shared+/// across concurrent requests. Never held across an `await`.+struct EnsembleState {+ /// Judge-win count per candidate model.+ wins: BTreeMap<String, u64>,+ /// Completed ensemble (exploration) turns.+ turns: u64,+ /// The model committed to once exploration is over; `None` while exploring.+ committed: Option<String>,+}++/// Ensemble router: fan out to candidates, judge the best, then commit.+pub struct EnsembleOrchAlgo {+ candidate_models: Vec<String>,+ judge_model: String,+ /// Number of ensemble turns to run before committing to the best model.+ /// `0` disables committing — the algorithm ensembles on every request.+ exploration_turns: u64,+ target_set: LlmTargetSet,+ state: Mutex<EnsembleState>,+}++impl EnsembleOrchAlgo {+ /// Create an ensemble over `candidate_models`, judged by `judge_model`,+ /// exploring for `exploration_turns` before committing to the winningest+ /// candidate (`0` = never commit, ensemble every request), routing among+ /// `target_set`. Wrap it in an [`Arc`](std::sync::Arc) and drive it with+ /// [`run`](libsy::Algorithm::run) or+ /// [`run_stream`](libsy::Algorithm::run_stream).+ pub fn new(+ candidate_models: Vec<String>,+ judge_model: impl Into<String>,+ exploration_turns: u64,+ target_set: LlmTargetSet,+ ) -> Self {+ Self {+ candidate_models,+ judge_model: judge_model.into(),+ exploration_turns,+ target_set,+ state: Mutex::new(EnsembleState {+ wins: BTreeMap::new(),+ turns: 0,+ committed: None,+ }),+ }+ }++ /// If exploration is over, return the committed model (committing lazily on+ /// the first post-exploration request). `None` means keep ensembling.+ ///+ /// The lock is taken and dropped here, never across an `await`. Under+ /// concurrency two requests may both commit; they pick the same model from+ /// the same tally (with a stable tie-break), so the result is identical.+ fn resolve_committed(&self) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {+ let mut state = self+ .state+ .lock()+ .map_err(|_| "ensemble state lock poisoned")?;+ if let Some(model) = &state.committed {+ return Ok(Some(model.clone()));+ }+ // `exploration_turns == 0` keeps the algorithm in ensemble mode forever.+ if self.exploration_turns > 0 && state.turns >= self.exploration_turns {+ let best = self.pick_best(&state.wins)?;+ state.committed = Some(best.clone());+ return Ok(Some(best));+ }+ Ok(None)+ }++ /// The candidate with the most judge-wins, breaking ties toward the earlier+ /// candidate (stable and deterministic). Errors only if there are no+ /// candidates configured.+ fn pick_best(+ &self,+ wins: &BTreeMap<String, u64>,+ ) -> Result<String, Box<dyn Error + Send + Sync>> {+ let mut best = self+ .candidate_models+ .first()+ .ok_or("no candidate models configured")?;+ let mut best_wins = wins.get(best).copied().unwrap_or(0);+ for model in &self.candidate_models[1..] {+ let w = wins.get(model).copied().unwrap_or(0);+ if w > best_wins {+ best = model;+ best_wins = w;+ }+ }+ Ok(best.clone())+ }++ /// Route a request to a single already-chosen model — the committed fast path.+ async fn route_committed(+ &self,+ driver: &Driver,+ request: Request,+ model: String,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {+ let target = self.target_set.get_target(&model)?;+ let decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {+ reasoning: format!(+ "committed to '{model}' after {} turns",+ self.exploration_turns+ ),+ selected_model: model.clone(),+ phase: EnsemblePhase::Committed,+ });+ let routed = Request {+ llm_request: LlmRequest {+ // The agent's inbound name rides through; the committed model is on+ // the decision, not stamped onto the request.+ inbound_model_name: request.llm_request.inbound_model_name,+ prompt: request.llm_request.prompt,+ },+ raw_request: request.raw_request,+ metadata: request.metadata,+ };+ let response = driver+ .call_llm_target(&target, routed, decision.clone())+ .await?;+ Ok((vec![decision], response))+ }++ /// One exploration turn: fan out to every candidate, judge the survivors,+ /// tally the winner, and return its response.+ async fn ensemble_turn(+ &self,+ driver: &Driver,+ request: Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {+ let user_prompt = request.llm_request.prompt.clone();+ // The agent's inbound name rides through every sub-call unchanged; the model+ // each call hits is carried by its decision, not stamped onto the request.+ let inbound = request.llm_request.inbound_model_name.clone();++ // Fan out to all candidates concurrently with the same user prompt. Each+ // call is annotated with its own candidate decision so the caller can see+ // which model an offloaded call targets.+ let mut candidate_decisions: Vec<Arc<dyn Decision>> = Vec::new();+ let mut calls = Vec::new();+ for model in &self.candidate_models {+ let target = self.target_set.get_target(model)?;+ let decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {+ selected_model: model.clone(),+ reasoning: format!("ensemble candidate '{model}'"),+ phase: EnsemblePhase::Candidate,+ });+ candidate_decisions.push(decision.clone());+ let call_request = Request {+ llm_request: LlmRequest {+ inbound_model_name: inbound.clone(),+ prompt: user_prompt.clone(),+ },+ raw_request: request.raw_request.clone(),+ metadata: request.metadata.clone(),+ };+ let model = model.clone();+ calls.push(async move {+ (+ model,+ driver+ .call_llm_target(&target, call_request, decision)+ .await,+ )+ });+ }+ let results = futures::future::join_all(calls).await;++ // Keep only successful responses, preserving candidate order. A failed+ // candidate is simply excluded from judging rather than failing the turn.+ let mut survivors: Vec<(String, Response)> = Vec::new();+ for (model, result) in results {+ if let Ok(response) = result {+ survivors.push((model, response));+ }+ }+ if survivors.is_empty() {+ return Err("all ensemble candidates failed".into());+ }++ // Pick the winner: judge only when there is a real choice to make.+ let (winner_model, winner_response, judge_decision) = if survivors.len() == 1 {+ let (model, response) = survivors+ .into_iter()+ .next()+ .ok_or("survivor unexpectedly missing")?;+ (model, response, None)+ } else {+ let judge_prompt = build_judge_prompt(&user_prompt, &survivors);+ let judge_target = self.target_set.get_target(&self.judge_model)?;+ let judge_decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {+ selected_model: self.judge_model.clone(),+ reasoning: format!("judging {} candidate responses", survivors.len()),+ phase: EnsemblePhase::Judge,+ });+ let judge_request = Request {+ llm_request: LlmRequest {+ inbound_model_name: inbound.clone(),+ prompt: judge_prompt,+ },+ raw_request: request.raw_request.clone(),+ metadata: request.metadata.clone(),+ };+ let judge_response = driver+ .call_llm_target(&judge_target, judge_request, judge_decision.clone())+ .await?;+ // Fail open: an unparseable pick falls back to the first response.+ let choice = parse_choice(&judge_response.llm_response.completion, survivors.len());+ let (model, response) = survivors+ .into_iter()+ .nth(choice)+ .ok_or("judge choice out of range")?;+ (model, response, Some(judge_decision))+ };++ // Record the win and advance the turn counter under the lock (not held+ // across any await).+ {+ let mut state = self+ .state+ .lock()+ .map_err(|_| "ensemble state lock poisoned")?;+ *state.wins.entry(winner_model.clone()).or_insert(0) += 1;+ state.turns += 1;+ }++ let winner_decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {+ reasoning: format!("judge selected '{winner_model}' as best response"),+ selected_model: winner_model,+ phase: EnsemblePhase::Winner,+ });++ // Trace order: [candidate calls..., judge?, winner].+ let mut trace = candidate_decisions;+ if let Some(judge_decision) = judge_decision {+ trace.push(judge_decision);+ }+ trace.push(winner_decision);+ Ok((trace, winner_response))+ }+}++/// Build the judge prompt. Responses are presented anonymously (no model names)+/// so the judge scores on content alone rather than model reputation.+fn build_judge_prompt(user_prompt: &str, survivors: &[(String, Response)]) -> String {+ let mut prompt = String::from(+ "You are an impartial judge. Choose which response best answers the user request.\n\n",+ );+ prompt.push_str("User request:\n");+ prompt.push_str(user_prompt);+ prompt.push_str("\n\n");+ for (i, (_model, response)) in survivors.iter().enumerate() {+ prompt.push_str(&format!(+ "Response {}:\n{}\n\n",+ i + 1,+ response.llm_response.completion+ ));+ }+ prompt.push_str(&format!(+ "Reply with only the number (1-{}) of the best response.",+ survivors.len()+ ));+ prompt+}++/// Parse the judge's 1-based pick into a 0-based index, failing open to the+/// first response. Reads the first run of digits in the reply, so "2" or+/// "Response 2 is best" both select index 1.+fn parse_choice(completion: &str, count: usize) -> usize {+ let mut digits = String::new();+ for c in completion.chars() {+ if c.is_ascii_digit() {+ digits.push(c);+ } else if !digits.is_empty() {+ break;+ }+ }+ match digits.parse::<usize>() {+ Ok(n) if n >= 1 && n <= count => n - 1,+ _ => 0,+ }+}++#[async_trait]+impl Algorithm for EnsembleOrchAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ // Fast path: exploration is over — route straight to the committed model;+ // otherwise run a full ensemble turn. Both return a decision trace plus the+ // final response.+ let (trace, response) = if let Some(model) = self.resolve_committed()? {+ self.route_committed(&driver, request, model).await?+ } else {+ self.ensemble_turn(&driver, request).await?+ };+ // Publish the trace to the stream (candidate..., judge?, winner). The+ // candidate decisions also rode along on their offloaded `CallLlm` steps.+ for decision in trace {+ driver.info(decision).await?;+ }+ Ok(response)+ }++ async fn process_signals(+ self: Arc<Self>,+ _signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Success is measured by the judge, not agent-system signals.+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};+ use std::sync::Mutex as StdMutex;++ /// Mock client that answers candidate calls with `answer from {model}` and,+ /// for the judge model, returns the 1-based number of the response whose+ /// content mentions `prefer` (so a test controls which candidate "wins").+ /// Records every model it was called with for call-count assertions.+ struct JudgingClient {+ judge_model: String,+ prefer: String,+ calls: Arc<StdMutex<Vec<String>>>,+ }++ #[async_trait]+ impl LlmClient for JudgingClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let name = routed.decision.selected_model().to_string();+ self.calls+ .lock()+ .map_err(|_| "lock poisoned")?+ .push(name.clone());+ let completion = if name == self.judge_model {+ judge_pick(&routed.request.llm_request.prompt, &self.prefer)+ } else {+ format!("answer from {name}")+ };+ Ok(Response {+ llm_response: LlmResponse {+ completion,+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ /// Scan a judge prompt for the `Response N:` whose body is `answer from+ /// {prefer}` and return `N` as a string; defaults to "1".+ fn judge_pick(prompt: &str, prefer: &str) -> String {+ let target_line = format!("answer from {prefer}");+ let mut current = 1u32;+ for line in prompt.lines() {+ if let Some(rest) = line.strip_prefix("Response ") {+ if let Ok(num) = rest.trim_end_matches(':').parse::<u32>() {+ current = num;+ }+ } else if line == target_line {+ return current.to_string();+ }+ }+ "1".to_string()+ }++ /// Build an ensemble algo over `candidates` + a judge, all backed by one+ /// judging client that prefers `prefer`. Returns the algo and the shared+ /// call log.+ fn algo(+ candidates: &[&str],+ judge: &str,+ prefer: &str,+ exploration_turns: u64,+ ) -> (EnsembleOrchAlgo, Arc<StdMutex<Vec<String>>>) {+ let calls = Arc::new(StdMutex::new(Vec::new()));+ let client = Arc::new(JudgingClient {+ judge_model: judge.to_string(),+ prefer: prefer.to_string(),+ calls: Arc::clone(&calls),+ }) as Arc<dyn LlmClient>;+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let mut targets: Vec<LlmTarget> = candidates.iter().map(|n| target(n)).collect();+ targets.push(target(judge));+ let algo = EnsembleOrchAlgo::new(+ candidates.iter().map(|s| s.to_string()).collect(),+ judge.to_string(),+ exploration_turns,+ LlmTargetSet::new(targets),+ );+ (algo, calls)+ }++ /// Build an ensemble algo whose candidate + judge targets all share `client`.+ /// One such algo models a single session's stateful router.+ fn algo_with_client(+ candidates: &[&str],+ judge: &str,+ exploration_turns: u64,+ client: Arc<dyn LlmClient>,+ ) -> EnsembleOrchAlgo {+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let mut targets: Vec<LlmTarget> = candidates.iter().map(|n| target(n)).collect();+ targets.push(target(judge));+ EnsembleOrchAlgo::new(+ candidates.iter().map(|s| s.to_string()).collect(),+ judge.to_string(),+ exploration_turns,+ LlmTargetSet::new(targets),+ )+ }++ fn request(prompt: &str) -> Request {+ Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: prompt.to_string(),+ },+ raw_request: None,+ metadata: None,+ }+ }++ /// Wrap an ensemble algo as `Arc<dyn Algorithm>` we can drive to completion.+ /// Reuse one handle across requests to exercise the algo's per-session state.+ fn orch(algo: EnsembleOrchAlgo) -> Arc<dyn Algorithm> {+ Arc::new(algo)+ }++ fn as_ensemble(+ d: &Arc<dyn Decision>,+ ) -> Result<&EnsembleDecision, Box<dyn Error + Send + Sync>> {+ d.as_any()+ .downcast_ref::<EnsembleDecision>()+ .ok_or_else(|| "expected an EnsembleDecision".into())+ }++ #[tokio::test]+ async fn exploration_turn_fans_out_judges_and_returns_the_winner(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Judge prefers b/model; it should win and be returned.+ let (algo, calls) = algo(&["a/model", "b/model"], "judge/haiku", "b/model", 100);+ let (trace, response) = orch(algo)+ .run(Context::default(), request("solve it"))+ .await?;+ assert_eq!(response.llm_response.completion, "answer from b/model");++ // Both candidates and the judge were called.+ let calls = calls.lock().map_err(|_| "lock poisoned")?;+ assert!(calls.contains(&"a/model".to_string()));+ assert!(calls.contains(&"b/model".to_string()));+ assert!(calls.contains(&"judge/haiku".to_string()));++ // Trace: [candidate a, candidate b, judge, winner].+ assert_eq!(trace.len(), 4);+ assert_eq!(as_ensemble(&trace[0])?.phase, EnsemblePhase::Candidate);+ assert_eq!(as_ensemble(&trace[2])?.phase, EnsemblePhase::Judge);+ let winner = as_ensemble(&trace[3])?;+ assert_eq!(winner.phase, EnsemblePhase::Winner);+ assert_eq!(winner.selected_model, "b/model");+ Ok(())+ }++ #[tokio::test]+ async fn commits_to_the_winningest_model_after_exploration(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Judge always prefers b/model over 2 exploration turns, so the algo+ // commits to b/model even though a/model is listed first.+ let (algo, calls) = algo(&["a/model", "b/model"], "judge/haiku", "b/model", 2);+ let orch = orch(algo);++ // Two exploration turns.+ orch.clone().run(Context::default(), request("t1")).await?;+ orch.clone().run(Context::default(), request("t2")).await?;+ let judge_calls_after_exploration = calls+ .lock()+ .map_err(|_| "lock poisoned")?+ .iter()+ .filter(|c| *c == "judge/haiku")+ .count();+ assert_eq!(judge_calls_after_exploration, 2);++ // Third request: committed fast path — routes straight to b/model with no+ // fan-out to a/model and no judge call.+ let (trace, response) = orch.clone().run(Context::default(), request("t3")).await?;+ assert_eq!(response.llm_response.completion, "answer from b/model");+ assert_eq!(trace.len(), 1);+ let decision = as_ensemble(&trace[0])?;+ assert_eq!(decision.phase, EnsemblePhase::Committed);+ assert_eq!(decision.selected_model, "b/model");++ let calls = calls.lock().map_err(|_| "lock poisoned")?;+ // Judge was not called again on the committed turn.+ assert_eq!(calls.iter().filter(|c| *c == "judge/haiku").count(), 2);+ // a/model was called only on the two exploration turns, not the third.+ assert_eq!(calls.iter().filter(|c| *c == "a/model").count(), 2);+ Ok(())+ }++ #[tokio::test]+ async fn single_candidate_skips_the_judge() -> Result<(), Box<dyn Error + Send + Sync>> {+ let (algo, calls) = algo(&["only/model"], "judge/haiku", "only/model", 100);+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;+ assert_eq!(response.llm_response.completion, "answer from only/model");+ // No judge call for a lone candidate.+ assert!(!calls+ .lock()+ .map_err(|_| "lock poisoned")?+ .contains(&"judge/haiku".to_string()));+ // Trace: [candidate, winner] — no judge entry.+ assert_eq!(trace.len(), 2);+ assert_eq!(as_ensemble(&trace[1])?.phase, EnsemblePhase::Winner);+ Ok(())+ }++ #[tokio::test]+ async fn zero_exploration_turns_never_commits() -> Result<(), Box<dyn Error + Send + Sync>> {+ // exploration_turns == 0 keeps ensembling forever.+ let (algo, calls) = algo(&["a/model", "b/model"], "judge/haiku", "b/model", 0);+ let orch = orch(algo);+ for _ in 0..3 {+ let (trace, _) = orch.clone().run(Context::default(), request("x")).await?;+ // Always a full ensemble turn (never a lone Committed decision).+ assert_eq!(+ as_ensemble(&trace[trace.len() - 1])?.phase,+ EnsemblePhase::Winner+ );+ }+ // Judge ran on every turn.+ assert_eq!(+ calls+ .lock()+ .map_err(|_| "lock poisoned")?+ .iter()+ .filter(|c| *c == "judge/haiku")+ .count(),+ 3+ );+ Ok(())+ }++ #[tokio::test]+ async fn all_candidates_failing_errors() -> Result<(), Box<dyn Error + Send + Sync>> {+ /// Client whose every call fails.+ struct FailingClient;+ #[async_trait]+ impl LlmClient for FailingClient {+ async fn call(+ &self,+ _routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ Err("upstream down".into())+ }+ }+ let client = Arc::new(FailingClient) as Arc<dyn LlmClient>;+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let algo = EnsembleOrchAlgo::new(+ vec!["a/model".to_string(), "b/model".to_string()],+ "judge/haiku",+ 100,+ LlmTargetSet::new(vec![+ target("a/model"),+ target("b/model"),+ target("judge/haiku"),+ ]),+ );+ assert!(orch(algo)+ .run(Context::default(), request("x"))+ .await+ .is_err());+ Ok(())+ }++ #[tokio::test]+ async fn process_signals_is_a_noop() -> Result<(), Box<dyn Error + Send + Sync>> {+ let (algo, _) = algo(&["a/model"], "judge/haiku", "a/model", 1);+ Arc::new(algo).process_signals(Signals {}).await?;+ Ok(())+ }++ #[test]+ fn parse_choice_reads_first_number_and_fails_open() {+ assert_eq!(parse_choice("2", 3), 1);+ assert_eq!(parse_choice("Response 3 is best", 3), 2);+ assert_eq!(parse_choice("the winner is 1", 3), 0);+ // Out of range and unparseable both fall open to the first response.+ assert_eq!(parse_choice("7", 3), 0);+ assert_eq!(parse_choice("none", 3), 0);+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn two_sessions_process_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {+ use std::time::Duration;+ use tokio::sync::Barrier;++ // Candidate calls block on a shared barrier; judge calls do not. Each session+ // serves its offloaded candidate calls one at a time (the step stream is+ // bounded), so a session has exactly one candidate call in flight at once. The+ // two sessions run concurrently, so the barrier releases only when both have a+ // candidate call pending. If the sessions were serialized, at most one call+ // could be pending, the barrier would never reach 2, and the test would time+ // out instead of passing.+ struct BarrierClient {+ barrier: Arc<Barrier>,+ judge_model: String,+ prefer: String,+ }++ #[async_trait]+ impl LlmClient for BarrierClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let name = routed.decision.selected_model().to_string();+ let completion = if name == self.judge_model {+ // Judge runs after the barrier releases; it must not wait.+ judge_pick(&routed.request.llm_request.prompt, &self.prefer)+ } else {+ // Hold every candidate call until all sessions have fanned out.+ self.barrier.wait().await;+ format!("answer from {name}")+ };+ Ok(Response {+ llm_response: LlmResponse {+ completion,+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ const SESSIONS: usize = 2;+ let barrier = Arc::new(Barrier::new(SESSIONS));+ let client = Arc::new(BarrierClient {+ barrier: barrier.clone(),+ judge_model: "judge/haiku".to_string(),+ prefer: "a/model".to_string(),+ }) as Arc<dyn LlmClient>;++ // Two independent sessions: separate algo instances, each with its own+ // per-session state, sharing only the backend client.+ let session_a: Arc<dyn Algorithm> = Arc::new(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 100,+ client.clone(),+ ));+ let session_b: Arc<dyn Algorithm> = Arc::new(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 100,+ client.clone(),+ ));++ let run = |session: Arc<dyn Algorithm>, prompt: &'static str| {+ tokio::spawn(async move {+ session+ .run(Context::default(), request(prompt))+ .await+ .map(|(_, response)| response.llm_response.completion)+ })+ };+ let handle_a = run(session_a, "from A");+ let handle_b = run(session_b, "from B");++ // The timeout converts a serialization deadlock into a failure, not a hang.+ let completion_a = tokio::time::timeout(Duration::from_secs(5), handle_a).await???;+ let completion_b = tokio::time::timeout(Duration::from_secs(5), handle_b).await???;+ assert_eq!(completion_a, "answer from a/model");+ assert_eq!(completion_b, "answer from a/model");+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn two_parallel_sessions_keep_independent_state(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Two sessions run concurrently with judges that prefer different models:+ // session A prefers a/model, session B prefers b/model. Each explores for+ // two turns then commits; because the win tally is per-session, they must+ // commit to *different* models — proving no state leaks between sessions.+ let (session_a, _) = algo(&["a/model", "b/model"], "judge/haiku", "a/model", 2);+ let (session_b, _) = algo(&["a/model", "b/model"], "judge/haiku", "b/model", 2);++ // Drive one session's three requests sequentially (so its two exploration+ // turns complete before the committing third), returning that third+ // request's winning model and decision phase.+ let drive = |session: Arc<dyn Algorithm>| {+ tokio::spawn(async move {+ session+ .clone()+ .run(Context::default(), request("t1"))+ .await?;+ session+ .clone()+ .run(Context::default(), request("t2"))+ .await?;+ let (trace, response) = session+ .clone()+ .run(Context::default(), request("t3"))+ .await?;+ let phase = trace+ .last()+ .and_then(|d| d.as_any().downcast_ref::<EnsembleDecision>())+ .map(|d| d.phase)+ .ok_or("missing final decision")?;+ Ok::<(String, EnsemblePhase), Box<dyn Error + Send + Sync>>((+ response.llm_response.completion,+ phase,+ ))+ })+ };+ // The two sessions run in parallel; each committed independently.+ let handle_a = drive(orch(session_a));+ let handle_b = drive(orch(session_b));+ let (completion_a, phase_a) = handle_a.await??;+ let (completion_b, phase_b) = handle_b.await??;++ assert_eq!(phase_a, EnsemblePhase::Committed);+ assert_eq!(completion_a, "answer from a/model");+ assert_eq!(phase_b, EnsemblePhase::Committed);+ assert_eq!(completion_b, "answer from b/model");+ Ok(())+ }+}diff --git a/crates/libsy-examples/src/lib.rs b/crates/libsy-examples/src/lib.rsnew file mode 100644index 000000000..91874debf--- /dev/null+++ b/crates/libsy-examples/src/lib.rs@@ -0,0 +1,17 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Reference algorithms for [`libsy`], kept out of the core crate but compiled and+//! tested here so they stay current. Each is a worked example of the+//! [`Algorithm`](libsy::Algorithm) trait; the `examples/` directory has runnable agents+//! that drive them.+//!+//! - [`rand::RandomOrchAlgo`] — uniform random over the target set (one call).+//! - [`llm_class::LlmClassifierOrchAlgo`] — classify with one model, then route to a+//! strong/weak model (multi-step).+//! - [`ensemble::EnsembleOrchAlgo`] — fan out to several models, judge, and commit+//! (stateful).++pub mod ensemble;+pub mod llm_class;+pub mod rand;diff --git a/crates/libsy-examples/src/llm_class.rs b/crates/libsy-examples/src/llm_class.rsnew file mode 100644index 000000000..94e252094--- /dev/null+++ b/crates/libsy-examples/src/llm_class.rs@@ -0,0 +1,349 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! LLM-classifier router built on the [`Algorithm`] interfaces.+//!+//! Unlike a local ML classifier (which scores a prompt in-process), an LLM+//! classifier needs its own model call to classify the request. On the new+//! interfaces this is just two ordinary `driver.call_llm_target`s inside one+//! `create_run_task`: first the classifier target (to get a score), then the+//! routed strong/weak target. The multi-step nature is invisible to the caller —+//! it is the algorithm's own control flow.++use std::error::Error;+use std::sync::Arc;++use async_trait::async_trait;++use libsy::{+ Algorithm, Context, Decision, Driver, LlmRequest, LlmTargetSet, Request, Response, Signals,+};++/// Preamble prepended to the user prompt when asking the classifier target for a+/// strong-win-rate score.+const CLASSIFIER_PROMPT_PREAMBLE: &str = "Rate how strongly this request needs a frontier model. \+ Reply with a single strong-win-rate score in [0, 1]:\n";++/// The tier a classifier score selected.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum ClassifierTier {+ Strong,+ Weak,+}++impl ClassifierTier {+ /// Stable string form of the tier, used in decision reasoning.+ pub fn as_str(self) -> &'static str {+ match self {+ ClassifierTier::Strong => "strong",+ ClassifierTier::Weak => "weak",+ }+ }+}++/// Decision produced at each step of the classifier flow. The classify step+/// leaves `score`/`tier` `None`; the routed step fills them in.+pub struct ClassifierDecision {+ /// The model this step selected (the classifier model, then the routed model).+ pub selected_model: String,+ /// Human-readable explanation of the step.+ pub reasoning: String,+ /// The classifier score, on the routed step; `None` on the classify step.+ pub score: Option<f64>,+ /// The tier chosen (strong/weak), on the routed step; `None` on the classify step.+ pub tier: Option<ClassifierTier>,+}++impl Decision for ClassifierDecision {+ fn selected_model(&self) -> &str {+ &self.selected_model+ }+ fn reasoning(&self) -> Option<&str> {+ Some(&self.reasoning)+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+}++/// LLM-classifier router: classify with one target, then route to strong/weak.+pub struct LlmClassifierOrchAlgo {+ classifier_model: String,+ strong_model: String,+ weak_model: String,+ threshold: f64,+ target_set: LlmTargetSet,+}++impl LlmClassifierOrchAlgo {+ /// Configure the classifier: the model that scores each request, the strong+ /// and weak models to route to, the score `threshold` at or above which the+ /// strong model is chosen, and the `target_set` to route among. That set must+ /// contain targets named `classifier_model`, `strong_model`, and `weak_model`.+ pub fn new(+ classifier_model: impl Into<String>,+ strong_model: impl Into<String>,+ weak_model: impl Into<String>,+ threshold: f64,+ target_set: LlmTargetSet,+ ) -> Self {+ Self {+ classifier_model: classifier_model.into(),+ strong_model: strong_model.into(),+ weak_model: weak_model.into(),+ threshold,+ target_set,+ }+ }+}++#[async_trait]+impl Algorithm for LlmClassifierOrchAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let user_prompt = request.llm_request.prompt.clone();+ // The agent's inbound name rides through unchanged on every sub-call; the+ // model each sub-call actually hits is carried by its decision instead.+ let inbound = request.llm_request.inbound_model_name.clone();++ // 1. Classify: call the classifier target with the score-eliciting prompt.+ let classifier_target = self.target_set.get_target(&self.classifier_model)?;+ let classify_request = Request {+ llm_request: LlmRequest {+ inbound_model_name: inbound.clone(),+ prompt: format!("{CLASSIFIER_PROMPT_PREAMBLE}{user_prompt}"),+ },+ raw_request: request.raw_request.clone(),+ metadata: request.metadata.clone(),+ };+ let classify_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {+ selected_model: self.classifier_model.clone(),+ reasoning: format!("classifying request via {}", self.classifier_model),+ score: None,+ tier: None,+ });+ driver.info(classify_decision.clone()).await?;+ let classify_response = driver+ .call_llm_target(&classifier_target, classify_request, classify_decision)+ .await?;+ let score = classify_response+ .llm_response+ .completion+ .trim()+ .parse::<f64>()+ .ok();++ // 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.+ let (tier, model) = match score {+ Some(s) if s >= self.threshold => (ClassifierTier::Strong, self.strong_model.clone()),+ Some(_) => (ClassifierTier::Weak, self.weak_model.clone()),+ None => (ClassifierTier::Strong, self.strong_model.clone()),+ };+ let routed_target = self.target_set.get_target(&model)?;+ let route_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {+ reasoning: format!(+ "classifier score {score:?} vs threshold {}; selected {model} ({})",+ self.threshold,+ tier.as_str()+ ),+ selected_model: model.clone(),+ score,+ tier: Some(tier),+ });+ let routed_request = Request {+ llm_request: LlmRequest {+ inbound_model_name: inbound,+ prompt: user_prompt,+ },+ raw_request: request.raw_request,+ metadata: request.metadata,+ };+ driver.info(route_decision.clone()).await?;+ driver+ .call_llm_target(&routed_target, routed_request, route_decision)+ .await+ }++ async fn process_signals(+ self: Arc<Self>,+ _signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Stateless classification; agent-system signals are ignored.+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};+ use std::sync::Mutex;++ /// Returns `score` for the classifier target, an answer tagged with the model+ /// otherwise; records the requests it saw so a test can inspect the classifier+ /// prompt.+ struct ScoringClient {+ classifier_model: String,+ score: String,+ seen: Arc<Mutex<Vec<Request>>>,+ }++ #[async_trait]+ impl LlmClient for ScoringClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let name = routed.decision.selected_model().to_string();+ let completion = if name == self.classifier_model {+ self.score.clone()+ } else {+ format!("answer from {name}")+ };+ self.seen+ .lock()+ .map_err(|_| "lock poisoned")?+ .push(routed.request);+ Ok(Response {+ llm_response: LlmResponse {+ completion,+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ /// Build a classifier algo whose three targets share a scoring client.+ fn algo(threshold: f64, score: &str) -> (LlmClassifierOrchAlgo, Arc<Mutex<Vec<Request>>>) {+ let seen = Arc::new(Mutex::new(Vec::new()));+ let client = Arc::new(ScoringClient {+ classifier_model: "router/classifier".to_string(),+ score: score.to_string(),+ seen: Arc::clone(&seen),+ }) as Arc<dyn LlmClient>;+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let target_set = LlmTargetSet::new(vec![+ target("router/classifier"),+ target("frontier/model"),+ target("cheap/model"),+ ]);+ let algo = LlmClassifierOrchAlgo {+ classifier_model: "router/classifier".to_string(),+ strong_model: "frontier/model".to_string(),+ weak_model: "cheap/model".to_string(),+ threshold,+ target_set,+ };+ (algo, seen)+ }++ fn request(prompt: &str) -> Request {+ Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: prompt.to_string(),+ },+ raw_request: None,+ metadata: None,+ }+ }++ /// Wrap a classifier algo as `Arc<dyn Algorithm>` we can drive to completion.+ fn orch(algo: LlmClassifierOrchAlgo) -> Arc<dyn Algorithm> {+ Arc::new(algo)+ }++ /// Downcast a trace entry to the concrete classifier decision.+ fn as_classifier(+ d: &Arc<dyn Decision>,+ ) -> Result<&ClassifierDecision, Box<dyn Error + Send + Sync>> {+ d.as_any()+ .downcast_ref::<ClassifierDecision>()+ .ok_or_else(|| "expected a ClassifierDecision".into())+ }++ #[tokio::test]+ async fn score_at_or_above_threshold_routes_strong() -> Result<(), Box<dyn Error + Send + Sync>>+ {+ let (algo, _) = algo(0.5, "0.9");+ let (trace, response) = orch(algo)+ .run(Context::default(), request("solve this proof"))+ .await?;+ assert_eq!(+ response.llm_response.completion,+ "answer from frontier/model"+ );+ // Trace: [classify, route].+ assert_eq!(trace[0].selected_model(), "router/classifier");+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.selected_model, "frontier/model");+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));+ assert_eq!(routed.score, Some(0.9));+ Ok(())+ }++ #[tokio::test]+ async fn score_below_threshold_routes_weak() -> Result<(), Box<dyn Error + Send + Sync>> {+ let (algo, _) = algo(0.5, "0.2");+ let (trace, response) = orch(algo)+ .run(Context::default(), request("say hello"))+ .await?;+ assert_eq!(response.llm_response.completion, "answer from cheap/model");+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.tier, Some(ClassifierTier::Weak));+ assert_eq!(routed.score, Some(0.2));+ Ok(())+ }++ #[tokio::test]+ async fn score_exactly_at_threshold_routes_strong() -> Result<(), Box<dyn Error + Send + Sync>>+ {+ let (algo, _) = algo(0.5, "0.5");+ let (_, response) = orch(algo)+ .run(Context::default(), request("borderline"))+ .await?;+ assert_eq!(+ response.llm_response.completion,+ "answer from frontier/model"+ );+ Ok(())+ }++ #[tokio::test]+ async fn unparseable_score_defaults_to_strong() -> Result<(), Box<dyn Error + Send + Sync>> {+ let (algo, _) = algo(0.5, "not-a-number");+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;+ assert_eq!(+ response.llm_response.completion,+ "answer from frontier/model"+ );+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));+ assert_eq!(routed.score, None);+ Ok(())+ }++ #[tokio::test]+ async fn classifier_prompt_includes_the_user_text() -> Result<(), Box<dyn Error + Send + Sync>>+ {+ let (algo, seen) = algo(0.5, "0.9");+ orch(algo)+ .run(Context::default(), request("prove it"))+ .await?;+ let seen = seen.lock().map_err(|_| "lock poisoned")?;+ // Two calls: the classifier (preamble + user text), then the routed model.+ assert_eq!(seen.len(), 2);+ assert!(seen[0].llm_request.prompt.contains("prove it"));+ assert!(seen[0].llm_request.prompt.contains("frontier model"));+ assert_eq!(seen[1].llm_request.prompt, "prove it");+ Ok(())+ }+}diff --git a/crates/libsy-examples/src/rand.rs b/crates/libsy-examples/src/rand.rsnew file mode 100644index 000000000..9e78f72f8--- /dev/null+++ b/crates/libsy-examples/src/rand.rs@@ -0,0 +1,232 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Random router built on the [`Algorithm`] interfaces.+//!+//! Selects one target from the set uniformly at random and calls it. This is the+//! simplest possible routing algorithm and the reference for the single-call+//! shape: one `driver.call_llm_target` inside `create_run_task`. (Weighted selection could+//! be layered on later; the set defines the candidates.)++use std::error::Error;+use std::sync::Arc;++use async_trait::async_trait;+use rand::seq::SliceRandom;++use libsy::{Algorithm, Context, Decision, Driver, LlmTargetSet, Request, Response, Signals};++/// Decision produced by [`RandomOrchAlgo`]: which target was chosen and why.+pub struct RandomDecision {+ /// The randomly selected target/model.+ pub selected_model: String,+ /// Human-readable explanation of the choice.+ pub reasoning: String,+}++impl Decision for RandomDecision {+ fn selected_model(&self) -> &str {+ &self.selected_model+ }+ fn reasoning(&self) -> Option<&str> {+ Some(&self.reasoning)+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+}++/// Uniform random router over a target set.+pub struct RandomOrchAlgo {+ target_set: LlmTargetSet,+}++impl RandomOrchAlgo {+ /// Create a router over `target_set`. Wrap it in an+ /// [`Arc`](std::sync::Arc) and drive it with+ /// [`run`](libsy::Algorithm::run) or+ /// [`run_stream`](libsy::Algorithm::run_stream).+ pub fn new(target_set: LlmTargetSet) -> Self {+ Self { target_set }+ }+}++#[async_trait]+impl Algorithm for RandomOrchAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ // Select a target uniformly at random. Scope the RNG so the non-Send+ // `ThreadRng` is dropped before the await below, keeping the returned+ // future `Send` (required by the `Algorithm` bound).+ let target = {+ let mut rng = rand::thread_rng();+ self.target_set+ .targets()+ .choose(&mut rng)+ .ok_or("no targets available")?+ .clone()+ };++ // Route by target semantic name; the caller's client (or offload host) maps+ // it to the provider model id when it serves or offloads the call.+ let selected = target.semantic_name.clone();+ let decision: Arc<dyn Decision> = Arc::new(RandomDecision {+ reasoning: format!("random routing selected target '{selected}'"),+ selected_model: selected,+ });++ // Publish the decision to the stream, then offload the call.+ driver.info(decision.clone()).await?;+ driver.call_llm_target(&target, request, decision).await+ }++ async fn process_signals(+ self: Arc<Self>,+ _signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Random routing is stateless, so agent-system signals are ignored.+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use libsy::{LlmClient, LlmRequest, LlmResponse, LlmTarget, Response, RoutedRequest};+ use std::collections::HashSet;++ /// Echoes back the target name it was called with, so a test can tell which+ /// target the algo selected.+ struct EchoClient;++ #[async_trait]+ impl LlmClient for EchoClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ Ok(Response {+ llm_response: LlmResponse {+ completion: routed.decision.selected_model().to_string(),+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ fn request() -> Request {+ Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: "hi".to_string(),+ },+ raw_request: None,+ metadata: None,+ }+ }++ /// Build a random-routing algorithm over `names`; every target echoes its name.+ fn orch(names: &[&str]) -> Arc<dyn Algorithm> {+ Arc::new(algo(names))+ }++ fn algo(names: &[&str]) -> RandomOrchAlgo {+ let targets: Vec<LlmTarget> = names+ .iter()+ .map(|name| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(Arc::new(EchoClient)),+ })+ .collect();+ RandomOrchAlgo::new(LlmTargetSet::new(targets))+ }++ #[tokio::test]+ async fn single_target_is_always_selected_and_called(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ let orch = orch(&["only/model"]);+ let (trace, response) = orch.clone().run(Context::default(), request()).await?;+ assert_eq!(response.llm_response.completion, "only/model");+ assert_eq!(trace.len(), 1);+ assert_eq!(trace[0].selected_model(), "only/model");+ Ok(())+ }++ #[tokio::test]+ async fn selected_target_is_in_the_set_and_matches_the_trace(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ let names = ["a/model", "b/model", "c/model"];+ let orch = orch(&names);+ for _ in 0..50 {+ let (trace, response) = orch.clone().run(Context::default(), request()).await?;+ let selected = response.llm_response.completion.clone();+ assert!(+ names.contains(&selected.as_str()),+ "selected {selected} not in target set"+ );+ // The trace records the same target that was actually called.+ assert_eq!(trace[0].selected_model(), selected.as_str());+ }+ Ok(())+ }++ #[tokio::test]+ async fn selection_covers_all_targets_over_many_runs(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ let orch = orch(&["a/model", "b/model"]);+ let mut seen = HashSet::new();+ for _ in 0..100 {+ let (_, response) = orch.clone().run(Context::default(), request()).await?;+ seen.insert(response.llm_response.completion);+ }+ // 100 uniform draws over two targets: both should appear (miss ~ 2^-99).+ assert_eq!(+ seen.len(),+ 2,+ "expected both targets to be selected, saw {seen:?}"+ );+ Ok(())+ }++ #[tokio::test]+ async fn empty_target_set_errors() {+ let orch = orch(&[]);+ assert!(orch+ .clone()+ .run(Context::default(), request())+ .await+ .is_err());+ }++ #[tokio::test]+ async fn process_signals_is_a_noop() -> Result<(), Box<dyn Error + Send + Sync>> {+ let algo = algo(&["only/model"]);+ Arc::new(algo).process_signals(Signals {}).await?;+ Ok(())+ }++ #[tokio::test]+ async fn decision_is_inspectable_and_downcasts() -> Result<(), Box<dyn Error + Send + Sync>> {+ let orch = orch(&["only/model"]);+ let (trace, _) = orch.clone().run(Context::default(), request()).await?;+ let decision = &trace[0];+ // Uniform, algo-agnostic access via the trait — no concrete type needed.+ assert_eq!(decision.selected_model(), "only/model");+ assert!(decision+ .reasoning()+ .unwrap_or_default()+ .contains("only/model"));+ // Escape hatch: downcast to the concrete decision when the algo is known.+ let concrete = decision+ .as_any()+ .downcast_ref::<RandomDecision>()+ .ok_or("expected a RandomDecision")?;+ assert_eq!(concrete.selected_model, "only/model");+ Ok(())+ }+}diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlnew file mode 100644index 000000000..dfd6bda0c--- /dev/null+++ b/crates/libsy/Cargo.toml@@ -0,0 +1,22 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "libsy"+version = "0.1.0"+description = "Switchyard library crate"+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true++[dependencies]+async-trait = "0.1"+serde_json = "1"+futures = "0.3"+tokio = { version = "1", features = ["full"] }+tokio-stream = "0.1"++[dev-dependencies]+tokio = { version = "1", features = ["macros", "rt"] }diff --git a/crates/libsy/README.md b/crates/libsy/README.mdnew file mode 100644index 000000000..652c1b43a--- /dev/null+++ b/crates/libsy/README.md@@ -0,0 +1,196 @@+# libsy — Switchyard-Lib++A lightweight, provider-agnostic library for multi-LLM agent optimization, with+**routing** as the first case. libsy *decides* how to serve each request — which+model(s) to call, in what order, how to combine them — and either makes the calls+itself or hands them back for you to make. It owns no HTTP client and no provider SDK,+so it drops into a proxy, gateway, or agent runtime.++## Example++Build a target set, pick an algorithm, run a request:++```rust+use libsy::llm_class::LlmClassifierOrchAlgo;+use libsy::{Algorithm, Context, LlmClient, LlmRequest, LlmTarget, LlmTargetSet, Request};+use std::sync::Arc;++// Targets the algorithm routes among, each backed by your LlmClient (see below).+let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn LlmClient>;+let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };++let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifierOrchAlgo::new(+ "classifier", "strong", "weak", 0.5,+ LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]),+));++let req = Request {+ llm_request: LlmRequest { inbound_model_name: "auto".into(), prompt: "explain tail latency".into() },+ raw_request: None,+ metadata: None,+};+let (trace, response) = algo.clone().run(Context::default(), req).await?; // calls in, trace + response out+println!("routed to {}", trace.last().unwrap().selected_model());+```++Runnable: [`research_agent`](../libsy-examples/examples/research_agent.rs) (in the `libsy-examples` crate).++## Requests & responses++```rust+pub struct Request {+ pub llm_request: LlmRequest, // normalized: { inbound_model_name, prompt }+ pub raw_request: Option<serde_json::Value>, // original provider body, forwarded verbatim if present+ pub metadata: Option<Metadata>, // correlation: session / agent / task / correlation_id / extra+}++pub struct Response {+ pub llm_response: LlmResponse, // normalized: { completion, raw_response? }+ pub metadata: Option<Metadata>,+}+```++## Targets and clients++An `LlmTarget` pairs a routing `semantic_name` with an optional `LlmClient`. Mapping that+name to a provider model id is the client's job, not the algorithm's — `LlmClient` is+meant to be implemented by you.++```rust+struct MyClient { /* http client, base url, key */ }++#[async_trait::async_trait]+impl LlmClient for MyClient {+ async fn call(&self, routed: RoutedRequest)+ -> Result<Response, Box<dyn std::error::Error + Send + Sync>> {+ let model = routed.decision.selected_model(); // the routed target — map it to a provider id+ // routed.request.llm_request.inbound_model_name is the agent's original name (not a call target)+ // ... POST to your endpoint, read the completion ...+ Ok(Response { llm_response: LlmResponse { completion, raw_response: None }, metadata: None })+ }+}+```++A `RoutedRequest` bundles the `request` with the routing `decision` and the target's+`default_client`; the model to call is `decision.selected_model()`, never a mutated+request field. `semantic_name` is the label an algorithm routes by; the client maps it to+the id it calls — they can differ (`"strong"` → `"openai/gpt-4o"`) or coincide.++## Running a request++Hold the algorithm as `Arc<dyn Algorithm>` and choose one of two entry points:++```rust+// run: libsy drives the request to completion, serving each call with the target's+// client, and returns (trace, response). Errors if a routed target has no client.+let (trace, response) = algo.clone().run(Context::default(), req).await?;++// run_stream: "ask, don't call" — you drive the stream and make the calls.+let stream = algo.clone().run_stream(Context::default(), req);+```++Under the hood every model call is *offloaded* to the request's `Step` stream; `run` is+the convenience that serves each one via the target's client. The step stream is bounded,+so pulling it paces the algorithm; each run is independent, so many run concurrently.++## Streaming — you own the model calls (`run_stream`)++`run_stream` yields `CallLlm` promises you fulfill with your own transport (or the call's+`default_client`), streams each `Decision` as it happens, and ends with `ReturnToAgent`.+Runnable: [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) (in the `libsy-examples` crate).++```rust+let stream = algo.clone().run_stream(Context::default(), req);+tokio::pin!(stream);+while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ let routed = call.get_routed()?.clone(); // which target, and its default client+ let response = call_model(routed.decision.selected_model(), &routed.request).await; // your real call+ call.respond(Ok(response))?; // or Err(..) to propagate a failure+ }+ Step::Decision(decision) => { /* decision.selected_model(), decision.reasoning() */ }+ Step::ReturnToAgent(response) => { /* done */ }+ }+}+```++## Building an algorithm (`Algorithm`)++Implement `Algorithm` to add a strategy. You write `create_run_task` — one call per+request; `run` / `run_stream` are provided and drive it. Make model calls on the `Driver`+you're handed, and publish a `Decision` for each so consumers (and clients) see *which*+model and *why*.++```rust+#[async_trait]+pub trait Algorithm: Send + Sync + 'static {+ // `self: Arc<Self>` (not `&mut`): one algorithm serves requests concurrently — use+ // interior mutability for state. Offload calls/decisions on `driver`.+ async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)+ -> Result<Response, Box<dyn Error + Send + Sync>>;+ async fn process_signals(self: Arc<Self>, signals: Signals)+ -> Result<(), Box<dyn Error + Send + Sync>>;+ // provided: run(ctx, request) -> (trace, response), run_stream(ctx, request) -> Stream<Step>+}++pub trait Decision: Send + Sync {+ fn selected_model(&self) -> &str; // the model chosen — the client's call target+ fn reasoning(&self) -> Option<&str>; // human-readable "why"+ fn as_any(&self) -> &dyn std::any::Any; // downcast to the concrete decision+}+```++Give it a `new(config.., target_set)` constructor and `Arc`-wrap it — there is no builder.+Example — the LLM classifier (classify, then route; full version in+[`libsy-examples/src/llm_class.rs`](../libsy-examples/src/llm_class.rs)):++```rust+#[async_trait]+impl Algorithm for LlmClassifierOrchAlgo {+ async fn create_run_task(self: Arc<Self>, _ctx: Context, driver: Driver, request: Request)+ -> Result<Response, Box<dyn Error + Send + Sync>> {+ // 1. Classify: ask the classifier target for a score.+ let classifier = self.target_set.get_target(&self.classifier_model)?;+ driver.info(classify_decision.clone()).await?;+ let score = driver.call_llm_target(&classifier, classify_req, classify_decision).await?+ .llm_response.completion.trim().parse::<f64>().ok();++ // 2. Route: strong if score >= threshold, else weak (fail open on None).+ let model = if score.map_or(true, |s| s >= self.threshold) { &self.strong_model } else { &self.weak_model };+ let routed = self.target_set.get_target(model)?;+ driver.info(route_decision.clone()).await?;+ driver.call_llm_target(&routed, routed_req, route_decision).await+ }++ async fn process_signals(self: Arc<Self>, _s: Signals) -> Result<(), Box<dyn Error + Send + Sync>> { Ok(()) }+}+```++## Explore++Reference algorithms and runnable agents live in the sibling+[`libsy-examples`](../libsy-examples) crate (kept out of `libsy` itself, but compiled and+tested — `cargo test -p libsy-examples`).++**Reference algorithms** — implementations to read and route with:++- [`RandomOrchAlgo`](../libsy-examples/src/rand.rs) — uniform random over the set (one call).+- [`LlmClassifierOrchAlgo`](../libsy-examples/src/llm_class.rs) — classify, then route+ strong/weak; fail open to strong.+- [`EnsembleOrchAlgo`](../libsy-examples/src/ensemble.rs) — stateful: fan out to+ candidates, judge the best, commit to the winner after N exploration turns.++**Runnable agents** (`cargo run -p libsy-examples --example <name>`):++- [`research_agent`](../libsy-examples/examples/research_agent.rs) — client-backed+ targets, `run` (libsy makes the calls).+- [`research_agent_core`](../libsy-examples/examples/research_agent_core.rs) — client-less+ targets, `run_stream` (the agent makes the calls).++## Not yet built++- **`Signals` events** — `process_signals` / `Signals` exist but carry nothing yet.+- **`Context` fields** — the per-request state carrier is an empty placeholder today.+- **Observability** — spans + a metrics sink (`Decision` is the hook).+- **Config-driven construction**, **typed errors** (vs `Box<dyn Error>`), **weighted random**.diff --git a/crates/libsy/src/driver.rs b/crates/libsy/src/driver.rsnew file mode 100644index 000000000..410f713d9--- /dev/null+++ b/crates/libsy/src/driver.rs@@ -0,0 +1,507 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! # driver — a type-erased promise-over-a-stream request pump+//!+//! [`TypeErasedDriver`] is the generic offload primitive; the crate root builds the+//! libsy-typed `Driver` on top of it. This module has no dependency on the rest of the+//! crate — the coupling is one-directional (`lib.rs` → `driver.rs`).+//!+//! A [`TypeErasedDriver`] lets a *producer* (e.g. a routing algorithm) fulfill+//! arbitrary requests by publishing promises onto a stream that a single *consumer*+//! drains. It is the type-erased generalization of the crate's `run_stream` offload:+//! instead of one fixed request/response shape, a producer calls+//! [`fulfill_request`](TypeErasedDriver::fulfill_request)+//! with *any* `REQ` and awaits *any* `RES`.+//!+//! - [`fulfill_request`](TypeErasedDriver::fulfill_request) enqueues a [`DriverStep::Request`]+//! carrying a [`DriverRequest`], then awaits the consumer's response.+//! - [`info`](TypeErasedDriver::info) pushes a fire-and-forget [`DriverStep::Info`] — no promise+//! to await.+//! - [`done`](TypeErasedDriver::done) emits the terminal [`DriverStep::Done`] with a final payload.+//! - [`stream`](TypeErasedDriver::stream) hands the single consumer the [`Stream`] of steps; for+//! each [`DriverStep::Request`] the consumer downcasts the request, computes a+//! response, and writes it back with [`DriverRequest::respond`].+//!+//! Payloads are erased to `Box<dyn Any + Send>`, so one `TypeErasedDriver` serves any request+//! type; the consumer downcasts to the concrete type it expects. `TypeErasedDriver` is `Clone`+//! (many producer tasks may call it concurrently — multi-producer), while the stream+//! is single-consumer. Each request rides its own `oneshot`, so concurrent+//! `fulfill_request` calls never cross responses.+//!+//! ## Pacing (bounded step channel)+//!+//! The step channel has capacity 1, so a producer cannot publish its next step until+//! the consumer has pulled the previous one. The consumer therefore *paces* the+//! algorithm: it advances one step for each `.next().await`. Every producer method is+//! `async` because publishing a step awaits channel capacity.+//!+//! ## Termination+//!+//! There is no explicit stop method — the consumer terminates by **dropping the+//! stream** (and any [`DriverRequest`] it is holding). The producer's next publish+//! (`fulfill_request`/`info`/`done`/`fail`) then resolves to `Err`, and a producer+//! awaiting a response sees `Err` once the promise it handed out is dropped. Either+//! way the algorithm unwinds cooperatively at its next driver interaction. Because the+//! producer runs on a task the driver does not own, hard cancellation (e.g. mid-compute+//! that never touches the driver) is the caller's concern — abort the producer task.++use std::{+ any::Any,+ error::Error,+ sync::{Arc, Mutex},+};++use futures::{Stream, StreamExt};+use tokio::sync::{mpsc, oneshot};+use tokio_stream::wrappers::ReceiverStream;++type BoxErr = Box<dyn Error + Send + Sync>;+type BoxAny = Box<dyn Any + Send>;+type StepResult = Result<DriverStep, BoxErr>;++/// One item on the stream returned by [`TypeErasedDriver::stream`].+pub enum DriverStep {+ /// A request awaiting a response. The consumer downcasts it and fulfills the+ /// paired promise with [`DriverRequest::respond`].+ Request(DriverRequest),+ /// A fire-and-forget payload from a producer; no response is expected.+ Info(BoxAny),+ /// A producer's terminal result. The consumer treats it as the last meaningful+ /// step (the stream itself closes when every [`TypeErasedDriver`] clone drops).+ Done(BoxAny),+}++/// The consumer-facing half of one [`TypeErasedDriver::fulfill_request`] call.+///+/// Yielded inside [`DriverStep::Request`]. The consumer reads the request via+/// [`request`](Self::request), does whatever work it names, and fulfills the promise+/// with [`respond`](Self::respond) — unblocking the producer's `fulfill_request`.+pub struct DriverRequest {+ request: BoxAny,+ // Fulfilled exactly once — `respond` consumes `self` to send, so no `Option`.+ tx: oneshot::Sender<Result<BoxAny, BoxErr>>,+}++impl DriverRequest {+ /// Borrow the request payload as `REQ`. Errors if the producer enqueued a+ /// different type than the consumer expected.+ pub fn request<REQ: Any>(&self) -> Result<&REQ, BoxErr> {+ self.request+ .downcast_ref::<REQ>()+ .ok_or_else(|| "driver: request type mismatch".into())+ }++ /// Fulfill the promise with a typed response, or an `Err` to propagate a failure+ /// back to the producer. Consumes `self`: a promise is fulfilled exactly once.+ pub fn respond<RES: Any + Send>(self, res: Result<RES, BoxErr>) -> Result<(), BoxErr> {+ // Erase the response so the single stream item type can carry any RES; the+ // producer downcasts it back in `fulfill_request`.+ let boxed: Result<BoxAny, BoxErr> = res.map(|r| Box::new(r) as BoxAny);+ self.tx+ .send(boxed)+ .map_err(|_| "driver: response receiver dropped".into())+ }+}++/// Internal shared state: the step channel plus the single, take-once receiver.+struct DriverInner {+ // Multi-producer: cloned into every `TypeErasedDriver`, so many tasks can enqueue steps.+ // Capacity 1: a producer blocks publishing its next step until the consumer pulls+ // the previous one, so the consumer paces the algorithm.+ step_tx: mpsc::Sender<StepResult>,+ // Single-consumer: taken out (once) by `stream`. `None` after the first take.+ step_rx: Mutex<Option<mpsc::Receiver<StepResult>>>,+}++/// A promise-over-a-stream request pump. See the [module docs](self) for the model.+///+/// Cheap to clone (shares one `Arc`); clone it to hand a producer handle to another+/// task. The consumer calls [`stream`](Self::stream) exactly once to drain steps.+#[derive(Clone)]+pub struct TypeErasedDriver {+ inner: Arc<DriverInner>,+}++impl TypeErasedDriver {+ /// Build an empty driver with its step channel ready. Take the consumer stream+ /// with [`stream`](Self::stream); enqueue work with the other methods.+ pub fn new() -> Self {+ let (step_tx, step_rx) = mpsc::channel(1);+ TypeErasedDriver {+ inner: Arc::new(DriverInner {+ step_tx,+ step_rx: Mutex::new(Some(step_rx)),+ }),+ }+ }++ /// Enqueue `req` as a [`DriverStep::Request`], await the consumer's response, and+ /// downcast it to `RES`. Errors if the stream is closed, the promise is dropped+ /// unfulfilled, the consumer responded with `Err`, or the response was not a `RES`.+ pub async fn fulfill_request<REQ, RES>(&self, req: REQ) -> Result<RES, BoxErr>+ where+ REQ: Any + Send + 'static,+ RES: Any + Send + 'static,+ {+ let (tx, rx) = oneshot::channel::<Result<BoxAny, BoxErr>>();+ let promise = DriverRequest {+ request: Box::new(req),+ tx,+ };+ self.inner+ .step_tx+ .send(Ok(DriverStep::Request(promise)))+ .await+ .map_err(|_| "driver: stream closed")?;++ // Outer error: the promise was dropped without a response. Inner error: the+ // consumer fulfilled it with an explicit `Err` — propagate it as-is.+ let response = match rx.await {+ Ok(result) => result?,+ Err(_) => return Err("driver: promise dropped without a response".into()),+ };+ response+ .downcast::<RES>()+ .map(|boxed| *boxed)+ .map_err(|_| "driver: response type mismatch".into())+ }++ /// Push a fire-and-forget [`DriverStep::Info`] payload; there is no promise to+ /// await for a response. Awaits channel capacity (the consumer pacing the stream)+ /// and errors only if the stream is closed.+ pub async fn info<INFO>(&self, info: INFO) -> Result<(), BoxErr>+ where+ INFO: Any + Send + 'static,+ {+ self.inner+ .step_tx+ .send(Ok(DriverStep::Info(Box::new(info))))+ .await+ .map_err(|_| "driver: stream closed".into())+ }++ /// Emit the terminal [`DriverStep::Done`] with a final payload. Does not close the+ /// stream (that happens when every `TypeErasedDriver` clone drops); the consumer treats it+ /// as the last meaningful step. Awaits channel capacity and errors only if the+ /// stream is closed.+ pub async fn done<T>(&self, payload: T) -> Result<(), BoxErr>+ where+ T: Any + Send + 'static,+ {+ self.inner+ .step_tx+ .send(Ok(DriverStep::Done(Box::new(payload))))+ .await+ .map_err(|_| "driver: stream closed".into())+ }++ /// Terminate the stream with an error item — the producer-side way to surface a+ /// failure to the consumer (mirrors how the crate's `run_stream` yields an `Err`+ /// step). Awaits channel capacity and errors only if the stream is+ /// already closed.+ pub async fn fail(&self, err: BoxErr) -> Result<(), BoxErr> {+ self.inner+ .step_tx+ .send(Err(err))+ .await+ .map_err(|_| "driver: stream closed".into())+ }++ /// Take the single consumer stream of [`DriverStep`]s. Callable once: a second+ /// call yields a one-item stream carrying an `Err`, since the receiver is gone.+ pub fn stream(&self) -> impl Stream<Item = Result<DriverStep, BoxErr>> {+ // Take the receiver out; `None` means already taken (or the lock was poisoned).+ let taken = match self.inner.step_rx.lock() {+ Ok(mut guard) => guard.take(),+ Err(_) => None,+ };+ match taken {+ Some(rx) => ReceiverStream::new(rx).left_stream(),+ None => futures::stream::once(async {+ Err::<DriverStep, BoxErr>("driver: stream already taken".into())+ })+ .right_stream(),+ }+ }+}++impl Default for TypeErasedDriver {+ fn default() -> Self {+ Self::new()+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use futures::StreamExt;++ #[tokio::test]+ async fn fulfill_request_round_trips_typed_values() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ // Producer asks for a u32 -> String on its own task.+ let producer = driver.clone();+ let handle =+ tokio::spawn(async move { producer.fulfill_request::<u32, String>(7u32).await });++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Request(promise) => {+ let req = *promise.request::<u32>()?;+ assert_eq!(req, 7);+ promise.respond::<String>(Ok(format!("got {req}")))?;+ }+ _ => return Err("expected a Request step".into()),+ }++ assert_eq!(handle.await??, "got 7");+ Ok(())+ }++ #[tokio::test]+ async fn info_pushes_a_typed_payload() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();+ driver.info(42u64).await?;++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Info(payload) => {+ let value = payload.downcast::<u64>().map_err(|_| "wrong info type")?;+ assert_eq!(*value, 42);+ }+ _ => return Err("expected an Info step".into()),+ }+ Ok(())+ }++ #[tokio::test]+ async fn done_emits_the_terminal_payload() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();+ driver.done("finished".to_string()).await?;++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Done(payload) => {+ let value = payload+ .downcast::<String>()+ .map_err(|_| "wrong done type")?;+ assert_eq!(*value, "finished");+ }+ _ => return Err("expected a Done step".into()),+ }+ Ok(())+ }++ #[tokio::test]+ async fn respond_error_propagates_to_the_producer() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ let producer = driver.clone();+ let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Request(promise) => {+ promise.respond::<u32>(Err("upstream failed".into()))?;+ }+ _ => return Err("expected a Request step".into()),+ }++ match handle.await? {+ Ok(_) => Err("expected the error to propagate".into()),+ Err(err) => {+ assert!(err.to_string().contains("upstream failed"));+ Ok(())+ }+ }+ }++ #[tokio::test]+ async fn response_type_mismatch_errors() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ // Producer expects a String back.+ let producer = driver.clone();+ let handle =+ tokio::spawn(async move { producer.fulfill_request::<u32, String>(1u32).await });++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Request(promise) => {+ // But the consumer responds with a u32.+ promise.respond::<u32>(Ok(99u32))?;+ }+ _ => return Err("expected a Request step".into()),+ }++ match handle.await? {+ Ok(_) => Err("expected a response type mismatch".into()),+ Err(err) => {+ assert!(err.to_string().contains("response type mismatch"));+ Ok(())+ }+ }+ }++ #[tokio::test]+ async fn request_downcast_to_wrong_type_errors() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ let producer = driver.clone();+ let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(5u32).await });++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ DriverStep::Request(promise) => {+ assert!(promise.request::<String>().is_err());+ // Unblock the producer so its task can finish.+ promise.respond::<u32>(Ok(5u32))?;+ }+ _ => return Err("expected a Request step".into()),+ }++ assert_eq!(handle.await??, 5);+ Ok(())+ }++ #[tokio::test]+ async fn closed_stream_errors_on_send() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ // Drop the consumer stream (and its receiver) before producing anything.+ drop(driver.stream());++ assert!(driver.fulfill_request::<u32, u32>(1u32).await.is_err());+ assert!(driver.info(1u32).await.is_err());+ assert!(driver.done(1u32).await.is_err());+ Ok(())+ }++ #[tokio::test]+ async fn promise_dropped_without_response_errors() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ let producer = driver.clone();+ let handle = tokio::spawn(async move { producer.fulfill_request::<u32, u32>(1u32).await });++ tokio::pin!(stream);+ match stream.next().await.ok_or("no step")?? {+ // Drop the promise without responding.+ DriverStep::Request(_promise) => {}+ _ => return Err("expected a Request step".into()),+ }++ assert!(handle.await?.is_err());+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_producers_do_not_cross_responses() -> Result<(), BoxErr> {+ const N: usize = 8;+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();++ // N producers each fulfill their own request concurrently.+ let mut handles = Vec::new();+ for i in 0..N {+ let producer = driver.clone();+ handles.push((+ i,+ tokio::spawn(async move { producer.fulfill_request::<usize, usize>(i).await }),+ ));+ }++ // Single consumer responds `req * 10` to each request.+ tokio::pin!(stream);+ let mut served = 0;+ while served < N {+ match stream.next().await.ok_or("stream ended early")?? {+ DriverStep::Request(promise) => {+ let req = *promise.request::<usize>()?;+ promise.respond::<usize>(Ok(req * 10))?;+ served += 1;+ }+ _ => return Err("expected a Request step".into()),+ }+ }++ // Each producer must see exactly its own response, not another's.+ for (i, handle) in handles {+ assert_eq!(handle.await??, i * 10);+ }+ Ok(())+ }++ #[tokio::test]+ async fn stream_taken_twice_yields_an_error_item() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let _first = driver.stream();+ let second = driver.stream();++ tokio::pin!(second);+ match second.next().await.ok_or("expected an item")? {+ Err(err) => {+ assert!(err.to_string().contains("already taken"));+ Ok(())+ }+ Ok(_) => Err("expected an error item".into()),+ }+ }++ #[tokio::test]+ async fn fail_surfaces_an_error_item_on_the_stream() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ let stream = driver.stream();+ driver.fail("kaboom".into()).await?;++ tokio::pin!(stream);+ match stream.next().await.ok_or("no item")? {+ Err(err) => {+ assert!(err.to_string().contains("kaboom"));+ Ok(())+ }+ Ok(_) => Err("expected an error item".into()),+ }+ }++ #[tokio::test]+ async fn dropping_the_stream_terminates_the_producer() -> Result<(), BoxErr> {+ let driver = TypeErasedDriver::new();+ // Box::pin so the stream is owned here and `drop` actually drops the receiver.+ // (`tokio::pin!` would rebind to a `Pin<&mut _>`, making `drop` a no-op.)+ let mut stream = Box::pin(driver.stream());++ // Producer publishes paced steps until the consumer goes away, then reports+ // how many it managed to send.+ let producer = driver.clone();+ let handle = tokio::spawn(async move {+ let mut sent = 0usize;+ while producer.info(sent).await.is_ok() {+ sent += 1;+ }+ sent+ });++ // Pace two steps, then terminate by dropping the stream.+ for _ in 0..2 {+ match stream.next().await.ok_or("stream ended early")?? {+ DriverStep::Info(_) => {}+ _ => return Err("expected an Info step".into()),+ }+ }+ drop(stream);++ // With the consumer gone, the producer's next publish errors and it stops.+ let sent = handle.await?;+ assert!(sent >= 2, "producer should have published the paced steps");+ Ok(())+ }+}diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsnew file mode 100644index 000000000..67b6536cc--- /dev/null+++ b/crates/libsy/src/lib.rs@@ -0,0 +1,784 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! # libsy — multi-LLM agent optimization (routing first)+//!+//! `libsy` decides, per request, *how* to serve an LLM call: which model(s) to+//! invoke, in what order, and how to combine the results. Routing is the first+//! and simplest case; the same interfaces also express classifier routing,+//! ensembles, cascades, and other optimizations. The library owns no HTTP client+//! and no provider SDK — it decides, and the host makes (or is asked to make) the+//! actual calls — so it embeds cleanly in a proxy, gateway, or agent runtime.+//!+//! ## The model+//!+//! - An [`Algorithm`] is the optimization *algorithm*. Its+//! [`create_run_task`](Algorithm::create_run_task) runs once per request+//! and makes as many model calls as it needs — via [`Driver::call_llm_target`], which look+//! like ordinary calls — publishes its [`Decision`]s with [`Driver::info`], and+//! returns the final [`Response`]. The provided+//! [`run_stream`](Algorithm::run_stream) drives that on its own task and hands+//! back a stream of [`Step`]s; [`run`](Algorithm::run) runs+//! it to completion with the targets' default clients.+//! - An [`LlmTarget`] names a routing target by its [`semantic_name`](LlmTarget::semantic_name).+//! Every call is *offloaded* to the request's stream as a [`Step::CallLlm`]; the+//! target's [`LlmClient`], if any, rides along as+//! [`RoutedRequest::default_client`] so the host can serve it by default or+//! override it (see below).+//!+//! ## Running a request+//!+//! Hold the algorithm as `Arc<dyn Algorithm>` and call one of two provided methods:+//!+//! - [`run`](Algorithm::run) — run to completion, serving each+//! offloaded call via its [`RoutedRequest::default_client`], and return the decision+//! trace plus the final [`Response`]. The simplest integration; use it when the+//! algorithm holds the model clients (it errors if a routed target has no client).+//! - [`run_stream`](Algorithm::run_stream) — return a stream of [`Step`]s. Each+//! model call is offloaded: the stream yields a [`Step::CallLlm`] carrying a promise;+//! the host performs the real model call (optionally via the promise's+//! `default_client`) and fulfills it with [`CallLlmRequest::respond`]. Decisions+//! arrive as [`Step::Decision`] as the algorithm makes them, and the run ends with a+//! [`Step::ReturnToAgent`] carrying the final response. The step stream is bounded,+//! so pulling it paces the algorithm one step at a time — an "ask, don't call" mode+//! that lets a host that owns its transport keep control of every call.+//!+//! ## Concurrency+//!+//! [`Algorithm::create_run_task`] takes `self: Arc<Self>`, so one shared+//! `Arc<dyn Algorithm>` (no lock) serves many requests in parallel. Each+//! [`run_stream`](Algorithm::run_stream) call builds its own [`Driver`], so+//! offloaded calls never cross between concurrent requests. An algorithm is+//! responsible for its own thread-safety — stateless (like the reference routers) or+//! interior mutability over just its own state.+//!+//! ## Reference algorithms+//!+//! Worked implementations — a random router, an LLM classifier, and a stateful+//! ensemble — plus runnable agents live in the `libsy-examples` crate.++mod driver;++use std::{error::Error, pin::Pin, sync::Arc};++use async_trait::async_trait;+use futures::{Stream, StreamExt};++use crate::driver::{DriverRequest, DriverStep, TypeErasedDriver};++/// Shorthand for the crate's boxed, thread-safe error type.+type BoxErr = Box<dyn Error + Send + Sync>;++/// A boxed, `Send` stream of [`Step`]s — the output of+/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps+/// `Arc<dyn Algorithm>` object-safe.+pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step, BoxErr>> + Send>>;++/// Correlation and routing metadata attached to a request or response.+///+/// All fields are optional; algorithms and observers use whichever are present+/// (e.g. to key per-session state or emit correlated telemetry). `extra_metadata`+/// is a free-form escape hatch for host-specific keys.+#[derive(Clone)]+pub struct Metadata {+ /// Stable id for a multi-request session/conversation.+ pub session_id: Option<String>,+ /// Id of the agent making the request.+ pub agent_id: Option<String>,+ /// Id of the task the request belongs to.+ pub task_id: Option<String>,+ /// External trace/request id for joining with the host's telemetry.+ pub correlation_id: Option<String>,+ /// Arbitrary host-defined key/value metadata.+ pub extra_metadata: Option<std::collections::BTreeMap<String, String>>,+}++/// The normalized model request an algorithm reasons over and hands to a target.+///+/// Deliberately minimal: a target model name and the user prompt. The full+/// provider-shaped request (messages, params, tools) rides on+/// [`Request::raw_request`] when a host needs to forward it losslessly.+#[derive(Clone)]+pub struct LlmRequest {+ /// The model to call. Algorithms rewrite this as they route.+ pub inbound_model_name: String,+ /// The user prompt an algorithm inspects (e.g. to classify) and sends.+ pub prompt: String,+}++/// A request entering the orchestrator: the normalized [`LlmRequest`] plus the+/// original provider payload and correlation [`Metadata`].+#[derive(Clone)]+pub struct Request {+ /// The normalized request an algorithm routes.+ pub llm_request: LlmRequest,+ /// The original provider-shaped request body, if the host wants to forward it+ /// verbatim (e.g. a proxy preserving messages/params). libsy does not read it.+ pub raw_request: Option<serde_json::Value>,+ /// Correlation metadata carried through the request.+ pub metadata: Option<Metadata>,+}++/// Agentic-stack events fed to an algorithm out of band via+/// [`Algorithm::process_signals`] (e.g. tool results, budget updates).+///+/// A placeholder today; a stateful algorithm can begin consuming signals as the+/// enum grows without changing the orchestrator contract.+#[derive(Clone)]+pub struct Signals {}++/// The neutral model response returned by a target.+#[derive(Clone)]+pub struct LlmResponse {+ /// The model's completion text — what an algorithm inspects (e.g. a+ /// classifier score) or returns.+ pub completion: String,+ /// Optional raw provider response body, so a host (e.g. a proxy) can forward+ /// the upstream response losslessly instead of rebuilding it from `completion`.+ pub raw_response: Option<serde_json::Value>,+}++/// A response leaving the orchestrator: the neutral [`LlmResponse`] plus optional+/// correlation [`Metadata`].+#[derive(Clone)]+pub struct Response {+ /// The neutral model response.+ pub llm_response: LlmResponse,+ /// Correlation metadata carried through the response.+ pub metadata: Option<Metadata>,+}++/// A decision/trace object produced by an algorithm.+///+/// Carried as a trait object (not a generic parameter) so a stream consumer can+/// inspect any algorithm's decision through this common interface without+/// knowing the concrete type. `as_any` is the escape hatch for a consumer that+/// *does* know the algo and wants to downcast to the concrete decision.+pub trait Decision: Send + Sync {+ /// The model this decision selected (e.g. the routed target's name).+ fn selected_model(&self) -> &str;+ /// A human-readable explanation of the decision, for logs and traces.+ fn reasoning(&self) -> Option<&str>;+ /// Downcast handle: a consumer that knows the algorithm can recover the+ /// concrete decision type via `as_any().downcast_ref::<ConcreteDecision>()`.+ fn as_any(&self) -> &dyn std::any::Any;+}++/// A request paired with the routing [`Decision`] that produced it — the unit an+/// [`LlmClient`] (or an offload host) is handed to serve.+///+/// The two model identifiers live in separate, unambiguous places: the model to+/// call is [`decision.selected_model()`](Decision::selected_model), while+/// `request.llm_request.inbound_model_name` is the *inbound* name the agent asked+/// for (libsy never overwrites it). A client maps `selected_model()` to the+/// provider model id it hits.+#[derive(Clone)]+pub struct RoutedRequest {+ /// The request to serve; its `inbound_model_name` is the agent's original name.+ pub request: Request,+ /// The routing decision behind this call; `selected_model()` is the model to hit.+ pub decision: Arc<dyn Decision>,+ /// The client that serves this call by default, or `None` when the routed target+ /// had no client. Rides along on the offloaded call so a host driving the stream+ /// can serve it by default or override it with its own transport.+ pub default_client: Option<Arc<dyn LlmClient>>,+}++/// The host-facing half of an offloaded model call, surfaced inside [`Step::CallLlm`].+///+/// Wraps a [`DriverRequest`] whose payload is a [`RoutedRequest`]. The host reads the+/// routed request ([`get_routed`](Self::get_routed)) and the decision behind it+/// ([`get_decision`](Self::get_decision)), performs (or delegates) the model call, and+/// fulfills it with [`respond`](Self::respond) — unblocking the algorithm's+/// [`Driver::call_llm`] on the other side.+pub struct CallLlmRequest {+ inner: DriverRequest,+}++impl CallLlmRequest {+ /// Wrap a driver request whose payload is a [`RoutedRequest`].+ fn new(inner: DriverRequest) -> Self {+ Self { inner }+ }++ /// The routed request the host should serve. Its+ /// [`default_client`](RoutedRequest::default_client) serves the call by default,+ /// and its `decision.selected_model()` names the model to hit. Errors if the+ /// promise payload was not a [`RoutedRequest`].+ pub fn get_routed(&self) -> Result<&RoutedRequest, BoxErr> {+ self.inner.request::<RoutedRequest>()+ }++ /// The model request to perform (the [`Request`] inside the routed request).+ pub fn get_request(&self) -> Result<&Request, BoxErr> {+ Ok(&self.get_routed()?.request)+ }++ /// The decision that led to this call — its `selected_model()` is the model to hit.+ pub fn get_decision(&self) -> Result<&dyn Decision, BoxErr> {+ Ok(self.get_routed()?.decision.as_ref())+ }++ /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to+ /// propagate a failed model call back to the algorithm. Consumes the promise: it+ /// can only be fulfilled once.+ pub fn respond(self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {+ self.inner.respond::<Response>(result)+ }+}++/// The offload channel handed to an algorithm's+/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls+/// with [`call_llm_target`](Self::call_llm_target) (or [`call_llm`](Self::call_llm)) and+/// publishes its [`Decision`]s with [`info`](Self::info); each call is offloaded to the+/// request's [`Step`] stream and awaits the consumer's response. The step channel is+/// bounded, so the consumer paces the algorithm one step at a time.+#[derive(Clone)]+pub struct Driver {+ driver: TypeErasedDriver,+}++impl Driver {+ /// Build an empty driver with its step channel ready. Created per call by+ /// [`run_stream`](Algorithm::run_stream).+ pub(crate) fn new() -> Self {+ Self {+ driver: TypeErasedDriver::new(),+ }+ }++ /// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the+ /// consumer's [`Response`]. Errors if the stream is closed or the call failed.+ pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response, BoxErr> {+ self.driver+ .fulfill_request::<RoutedRequest, Response>(routed)+ .await+ }++ /// Offload a call to `target`: pair `request` with `decision` and the target's+ /// default client into a [`RoutedRequest`], then publish it (see+ /// [`call_llm`](Self::call_llm)). The convenience most algorithms use;+ /// `decision.selected_model()` names the model to hit, and `request`'s+ /// `inbound_model_name` is left untouched.+ pub async fn call_llm_target(+ &self,+ target: &LlmTarget,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, BoxErr> {+ self.call_llm(RoutedRequest {+ request,+ decision,+ default_client: target.llm_client.clone(),+ })+ .await+ }++ /// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.+ pub async fn info(&self, decision: Arc<dyn Decision>) -> Result<(), BoxErr> {+ self.driver.info(decision).await+ }++ /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream+ /// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)+ /// when the algorithm finishes.+ pub(crate) async fn finish(&self, result: Result<Response, BoxErr>) -> Result<(), BoxErr> {+ match result {+ Ok(response) => self.driver.done(response).await,+ Err(err) => self.driver.fail(err).await,+ }+ }++ /// Transform the raw driver stream into a stream of [`Step`]s. Internal: the+ /// consumer stream is taken (once) by [`run_stream`](Algorithm::run_stream). A+ /// payload that does not match the expected type for its step becomes an `Err` item.+ pub(crate) fn stream(&self) -> impl Stream<Item = Result<Step, BoxErr>> {+ self.driver.stream().map(|item| match item? {+ DriverStep::Request(req) => Ok(Step::CallLlm(CallLlmRequest::new(req))),+ DriverStep::Info(payload) => payload+ .downcast::<Arc<dyn Decision>>()+ .map(|decision| Step::Decision(*decision))+ .map_err(|_| "driver: info payload was not a Decision".into()),+ DriverStep::Done(payload) => payload+ .downcast::<Response>()+ .map(|response| Step::ReturnToAgent(*response))+ .map_err(|_| "driver: done payload was not a Response".into()),+ })+ }+}++impl Default for Driver {+ fn default() -> Self {+ Self::new()+ }+}++/// Per-request state threaded to an algorithm alongside its [`Driver`]. A placeholder+/// for cross-cutting state (correlation ids, budgets, deadlines) an algorithm will+/// read; empty today. It does not carry the offload driver, so it is safe to share.+#[derive(Clone, Default)]+pub struct Context {}++impl Context {+ /// Build an empty context.+ pub fn new() -> Self {+ Self {}+ }+}++/// One item in the stream returned by [`Driver::stream`] / [`Algorithm::run_stream`].+pub enum Step {+ /// The algorithm needs this model call performed. The host serves it (optionally+ /// via [`RoutedRequest::default_client`]) and fulfills it with+ /// [`CallLlmRequest::respond`].+ CallLlm(CallLlmRequest),+ /// A routing decision the algorithm made, published via [`Driver::info`] as it+ /// happens (rather than collected into a trace returned at the end).+ Decision(Arc<dyn Decision>),+ /// The algorithm finished with its final response — the last step of a run.+ ReturnToAgent(Response),+}++/// Performs the actual model call for a target. This is the one piece of I/O+/// `libsy` does not own — a host implements it over its own transport (HTTP SDK,+/// in-process model, mock). It serves a call the stream consumer chose not to+/// override, reached as [`RoutedRequest::default_client`] (see [`Algorithm::run_stream`]).+#[async_trait]+pub trait LlmClient: Send + Sync {+ /// Serve `routed`, returning the model's response. Call the model named by+ /// [`routed.decision.selected_model()`](Decision::selected_model) — the target+ /// the algorithm routed to — mapping it to whatever provider model id this+ /// client hits. `routed.request.llm_request.inbound_model_name` is the agent's+ /// original name, carried through for reference, not a call target.+ async fn call(&self, request: RoutedRequest) -> Result<Response, Box<dyn Error + Send + Sync>>;+}++/// A named routing target: a `semantic_name` an algorithm routes by, and an optional+/// [`LlmClient`] to serve its calls. An algorithm hands a target to+/// [`Driver::call_llm_target`]; the client rides along as+/// [`RoutedRequest::default_client`] for the stream consumer to serve or override.+#[derive(Clone)]+pub struct LlmTarget {+ /// The routing label an algorithm selects this target by — a logical tier like+ /// `"strong"`, or the model id when they coincide. Mapping it to a provider model+ /// id is the client's concern, never the algorithm's.+ pub semantic_name: String,+ /// The client that serves this target's calls by default, or `None` (then the+ /// stream consumer must serve them).+ pub llm_client: Option<Arc<dyn LlmClient>>,+}++/// The set of targets an algorithm may route among. An algorithm is constructed+/// with one and picks targets by position ([`targets`](Self::targets)) or by name+/// ([`get_target`](Self::get_target)).+#[derive(Clone)]+pub struct LlmTargetSet {+ targets: Vec<LlmTarget>,+}++impl LlmTargetSet {+ /// Build a target set from a list of targets.+ pub fn new(targets: Vec<LlmTarget>) -> Self {+ Self { targets }+ }++ /// All targets in the set — e.g. for an algorithm to select among.+ pub fn targets(&self) -> &[LlmTarget] {+ &self.targets+ }++ /// Look up a target by name; errors if no target has that name.+ pub fn get_target(&self, name: &str) -> Result<LlmTarget, Box<dyn Error + Send + Sync>> {+ self.targets+ .iter()+ .find(|t| t.semantic_name == name)+ .cloned()+ .ok_or(format!("Target {} not found", name).into())+ }+}++/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);+/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)+/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).+///+/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across+/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms+/// (the reference routers) get this for free; a stateful one uses interior mutability+/// over just its own state.+#[async_trait]+pub trait Algorithm: Send + Sync + 'static {+ /// Run one request to completion: make model calls with [`Driver::call_llm_target`],+ /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].+ /// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)+ /// drive it. `ctx` carries cross-cutting request state (empty today).+ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>>;++ /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The+ /// reference algorithms ignore signals; a stateful algorithm updates its own+ /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.+ async fn process_signals(+ self: Arc<Self>,+ signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>>;++ /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its+ /// own task; drive the stream: serve each [`Step::CallLlm`] (via its+ /// [`default_client`](RoutedRequest::default_client) or your own transport) and read+ /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is+ /// bounded, so pulling paces the algorithm; each call is independent, so many run+ /// concurrently.+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {+ // This call's own driver: take its consumer stream, hand a producer-side clone to+ // the algorithm task, and keep one to emit the terminal step. The task blocks+ // publishing a step until the consumer pulls the previous one.+ let driver = Driver::new();+ let stream = driver.stream();+ tokio::spawn(async move {+ let outcome = self.create_run_task(ctx, driver.clone(), request).await;+ let _ = driver.finish(outcome).await;+ });+ Box::pin(stream)+ }++ /// Run one request to completion, serving each offloaded call with its+ /// [`RoutedRequest::default_client`], and return the decision trace plus the final+ /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,+ /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model+ /// clients and the host wants the answer (and the decisions behind it); drive+ /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. Errors+ /// if a routed target has no client to serve its call, or the algorithm fails.+ async fn run(+ self: Arc<Self>,+ ctx: Context,+ request: Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {+ let stream = self.run_stream(ctx, request);+ tokio::pin!(stream);+ let mut trace: Vec<Arc<dyn Decision>> = Vec::new();+ while let Some(item) = stream.next().await {+ match item? {+ Step::CallLlm(call) => {+ // Serve the call with the target's default client, or error if the+ // routed target had none.+ let routed = call.get_routed()?.clone();+ let client = routed.default_client.clone().ok_or_else(|| {+ format!(+ "run: target '{}' has no client to serve the call",+ routed.decision.selected_model()+ )+ })?;+ call.respond(client.call(routed).await)?;+ }+ Step::Decision(decision) => trace.push(decision),+ // The terminal step: return as soon as the algorithm finishes, rather+ // than draining the stream until it closes.+ Step::ReturnToAgent(response) => return Ok((trace, response)),+ }+ }+ Err("run: stream ended without a final response".into())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use futures::StreamExt;++ /// Mock client that echoes back the target name it was called with.+ struct EchoClient;++ #[async_trait]+ impl LlmClient for EchoClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ // Echo back the model the algorithm routed to (the decision's selection).+ Ok(Response {+ llm_response: LlmResponse {+ completion: routed.decision.selected_model().to_string(),+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ /// Trivial decision + algo used only to exercise the orchestrator: calls the+ /// first target and returns its response with a one-item trace.+ struct TestDecision {+ model: String,+ }++ impl Decision for TestDecision {+ fn selected_model(&self) -> &str {+ &self.model+ }+ fn reasoning(&self) -> Option<&str> {+ None+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+ }++ struct TestAlgo {+ target_set: LlmTargetSet,+ }++ #[async_trait]+ impl Algorithm for TestAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let target = self+ .target_set+ .targets()+ .first()+ .ok_or("no targets")?+ .clone();+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {+ model: target.semantic_name.clone(),+ });+ driver.info(decision.clone()).await?;+ driver.call_llm_target(&target, request, decision).await+ }++ async fn process_signals(+ self: Arc<Self>,+ _signals: Signals,+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ Ok(())+ }+ }++ /// Build a shared `TestAlgo` over the given target set.+ fn orch(target_set: LlmTargetSet) -> Arc<dyn Algorithm> {+ Arc::new(TestAlgo { target_set })+ }++ fn request() -> Request {+ Request {+ llm_request: LlmRequest {+ inbound_model_name: "auto".to_string(),+ prompt: "hi".to_string(),+ },+ raw_request: None,+ metadata: None,+ }+ }++ /// `(name, has_client)` — `has_client: false` builds a target with no default client.+ fn target_set(names: &[(&str, bool)]) -> LlmTargetSet {+ let targets = names+ .iter()+ .map(|(name, has_client)| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: has_client.then(|| Arc::new(EchoClient) as Arc<dyn LlmClient>),+ })+ .collect();+ LlmTargetSet::new(targets)+ }++ #[tokio::test]+ async fn run_offloads_via_promise_then_returns_to_agent(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target -> its call is offloaded via a promise the+ // orchestrator surfaces as a `CallLlm` step for us to fulfill.+ let stream =+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut saw_call = false;+ let mut final_completion = None;+ while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ saw_call = true;+ // The decision rode along with the promise.+ assert_eq!(call.get_decision()?.selected_model(), "offload/model");+ // Fulfilling the promise is the "real" model call the caller makes.+ call.respond(Ok(Response {+ llm_response: LlmResponse {+ completion: "fulfilled".to_string(),+ raw_response: None,+ },+ metadata: None,+ }))?;+ }+ Step::Decision(decision) => {+ assert_eq!(decision.selected_model(), "offload/model");+ }+ Step::ReturnToAgent(response) => {+ final_completion = Some(response.llm_response.completion);+ }+ }+ }++ assert!(saw_call, "expected a CallLlm step before ReturnToAgent");+ assert_eq!(+ final_completion.ok_or("no ReturnToAgent step")?,+ "fulfilled"+ );+ Ok(())+ }++ #[tokio::test]+ async fn client_backed_target_offloads_with_a_default_client(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Every call now offloads to the stream; a client-backed target rides its+ // client along as `default_client` so the consumer can serve it by default.+ let stream =+ orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut final_completion = None;+ while let Some(step) = stream.next().await {+ match step? {+ Step::CallLlm(call) => {+ let routed = call.get_routed()?.clone();+ let client = routed+ .default_client+ .clone()+ .ok_or("expected a default client")?;+ let result = client.call(routed).await;+ call.respond(result)?;+ }+ Step::Decision(_) => {}+ Step::ReturnToAgent(response) => {+ final_completion = Some(response.llm_response.completion);+ }+ }+ }++ // EchoClient echoes the model name back as the completion.+ assert_eq!(final_completion.ok_or("no ReturnToAgent")?, "direct/model");+ Ok(())+ }++ #[tokio::test]+ async fn run_returns_the_response_when_all_targets_have_clients(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // Every target has a client, so run serves every call via the+ // default client and returns the trace + final response.+ let (trace, response) = orch(target_set(&[("direct/model", true)]))+ .run(Context::default(), request())+ .await?;+ // TestAlgo calls the first target; EchoClient echoes its name.+ assert_eq!(response.llm_response.completion, "direct/model");+ assert_eq!(trace[0].selected_model(), "direct/model");+ Ok(())+ }++ #[tokio::test]+ async fn run_errors_when_a_target_lacks_a_client() -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target has no default client to serve its offloaded call, so+ // driving it to completion errors.+ assert!(orch(target_set(&[("offload/model", false)]))+ .run(Context::default(), request())+ .await+ .is_err());+ Ok(())+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn requests_are_processed_in_parallel() -> Result<(), Box<dyn Error + Send + Sync>> {+ use std::time::Duration;+ use tokio::sync::Barrier;++ const N: usize = 4;++ // A client that blocks until all N concurrent calls have arrived. If+ // requests were serialized (one algorithm behind a `Mutex`), only one+ // call could be in flight, the barrier would never reach N, and the test+ // would time out. It passes only because the shared algorithm is driven+ // concurrently across requests.+ struct BarrierClient {+ barrier: Arc<Barrier>,+ }++ #[async_trait]+ impl LlmClient for BarrierClient {+ async fn call(+ &self,+ routed: RoutedRequest,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ self.barrier.wait().await;+ Ok(Response {+ llm_response: LlmResponse {+ completion: routed.decision.selected_model().to_string(),+ raw_response: None,+ },+ metadata: None,+ })+ }+ }++ let barrier = Arc::new(Barrier::new(N));+ let targets = LlmTargetSet::new(vec![LlmTarget {+ semantic_name: "m".to_string(),+ llm_client: Some(Arc::new(BarrierClient {+ barrier: barrier.clone(),+ })),+ }]);+ // One shared algorithm driven by many concurrent requests.+ let algo = orch(targets);++ let mut handles = Vec::new();+ for _ in 0..N {+ let algo = algo.clone();+ handles.push(tokio::spawn(async move {+ algo.run(Context::default(), request())+ .await+ .map(|(_, response)| response.llm_response.completion)+ }));+ }++ for handle in handles {+ // The timeout turns a serialization deadlock into a failure, not a hang.+ let completion = tokio::time::timeout(Duration::from_secs(5), handle).await???;+ assert_eq!(completion, "m");+ }+ Ok(())+ }++ #[tokio::test]+ async fn offload_error_propagates_back_to_the_algorithm(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // A client-less target offloads its call; we fulfill the promise with an+ // Err, which must flow back through `call_llm_target` into the algorithm and+ // out as an error step — not a response.+ let stream =+ orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut saw_error = false;+ while let Some(step) = stream.next().await {+ match step {+ Ok(Step::CallLlm(call)) => {+ call.respond(Err("upstream model call failed".into()))?;+ }+ Ok(Step::Decision(_)) => {}+ Ok(Step::ReturnToAgent(..)) => {+ return Err("expected the offload error to propagate, got a response".into());+ }+ Err(err) => {+ // The algorithm's `call_llm_target` saw the error via the promise.+ assert!(err.to_string().contains("upstream model call failed"));+ saw_error = true;+ }+ }+ }++ assert!(saw_error, "expected an error step");+ Ok(())+ }+}