Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): add typed decision calls to the driver (#894)
nachiketb-nvidiae9ef1d17bba0
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 939599b81..a3dfc339a 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -27,7 +27,8 @@ use http::StatusCode;use parking_lot::Mutex;use serde_json::{Value, json};use switchyard_libsy::{- Algorithm, CallModel, LibsyError, OutcomeMetadata, Result, RoutingOutcome, RuntimeModels, drive,+ Algorithm, Call, CallDecision, LibsyError, OutcomeMetadata, Result, RoutingOutcome,+ RuntimeModels, drive,};use switchyard_protocol::{AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,@@ -151,6 +152,14 @@ pub async fn decide(Ok(outcome)}+async fn unsupported_decision(call: CallDecision) -> Result<()> {+ let model = call.model.clone();+ call.respond(Err(LibsyError::client_call(+ model,+ LlmClientError::General("decision calls are not supported by this client".to_string()),+ )))+}+/// Emits completed routing calls after the outcome reveals whether one response became the answer.fn emit_routing_observations(observer: &Option<RunObserver>,@@ -173,12 +182,17 @@ fn emit_routing_observations(/// Serve one offloaded call and fulfill its promise.///-/// A client failure stops the driver unless this call opts into algorithm-level recovery.+/// LLM failures stop the run unless the call enables recovery. Unsupported decisions+/// return an error to the algorithm.async fn serve(clients: ClientRouter,- call: CallModel,+ call: Call,observations: Option<Arc<Mutex<Vec<LlmCallObservation>>>>,) -> Result<()> {+ let call = match call {+ Call::Model(call) => *call,+ Call::Decision(call) => return unsupported_decision(*call).await,+ };let observe = |observation| {if let Some(observations) = &observations {observations.lock().push(observation);diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 30adc407c..820c762c6 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -1619,6 +1619,7 @@ async fn failed_call_records_metrics_without_error_details() -> switchyard_libsyOk(Step::CallModel(call)) => {call.respond(Err(test_error("synthetic upstream failure")))?;}+ Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),Ok(Step::Done(_)) => {return Err(test_error("expected the failed call to fail the run"));}diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rsindex 10bb968d4..8bf901595 100644--- a/crates/libsy/src/algorithms/advisor_gate/tests.rs+++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs@@ -16,8 +16,8 @@ use switchyard_protocol::{use super::transcript::{NO_TEXT_PLACEHOLDER, TRUNCATION_MARKER, middle_drop};use super::*;-use crate::RuntimeModels;-use crate::core::testing::{Serve, reply, test_drive_with_models};+use crate::core::testing::{Serve, reply, serve_decision, test_drive_with_models};+use crate::{Call, RuntimeModels};const EXECUTOR: &str = "executor";const ADVISOR: &str = "advisor";@@ -343,6 +343,10 @@ async fn calls_preserve_candidates_and_attribute_the_serving_executor() {let outcome = crate::drive(gate, task_request(), Arc::new(models), move |call| {let observed = Arc::clone(&observed);async move {+ let call = match call {+ Call::Model(call) => *call,+ Call::Decision(call) => return serve_decision(*call).await,+ };let candidates = call.models.clone();observed.lock().push(candidates.clone());let (text, served) = if candidates[0] == target(EXECUTOR) {diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rsindex 76c77d76b..7e3d15b83 100644--- a/crates/libsy/src/algorithms/escalation.rs+++ b/crates/libsy/src/algorithms/escalation.rs@@ -444,7 +444,7 @@ mod tests {use super::*;use crate::algorithms::llm_class::{LlmClassifierConfig, LlmTaskClassifier};use crate::algorithms::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;- use crate::core::testing::{Serve, reply, test_drive_with_models};+ use crate::core::testing::{Serve, reply, serve_decision, test_drive_with_models};/// A queue of replies, drained in order.struct Queue(Mutex<VecDeque<String>>);@@ -624,6 +624,10 @@ mod tests {move |call| {let serve = Arc::clone(&routing_serve);async move {+ let call = match call {+ crate::Call::Model(call) => *call,+ crate::Call::Decision(call) => return serve_decision(*call).await,+ };let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;let request = call.request.clone();let response = servediff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 5c3ebda92..aabab68a3 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -1,8 +1,8 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! The [`Algorithm`] trait and its [`Driver`] — the orchestration contract every-//! algorithm implements and the offload channel it uses for routing-time model calls.+//! Routing algorithms request external work through [`Driver`] and receive typed replies.+//! Hosts execute that work, keeping routing policy independent of transport.use std::{collections::HashMap, future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc,@@ -24,7 +24,9 @@ use tracing::Instrument;/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and/// [`switchyard_protocol::LlmResponse`] carries either a live/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.-use switchyard_protocol::{Category, ModelId, Request, Response};+use switchyard_protocol::{+ Category, DecisionRequest, DecisionResponse, ModelId, Request, Response,+};use crate::{DriverError, LibsyError, Result, observability};@@ -158,6 +160,31 @@ impl Drop for CallModel {}}+/// A host-owned decision call. Dropping it without replying yields [`DriverError::ResponseDropped`].+pub struct CallDecision {+ /// Algorithm name for attributing host telemetry.+ pub algorithm: String,+ /// Its model is set to [`Self::model`] before dispatch.+ pub request: DecisionRequest,+ /// Target ID for client lookup.+ pub model: ModelId,+ reply: oneshot::Sender<Result<DecisionResponse>>,+}++impl CallDecision {+ /// Return a response or provider error to the algorithm so it can continue or fall back.+ pub fn respond(self, result: Result<DecisionResponse>) -> Result<()> {+ self.reply+ .send(result)+ .map_err(|_| DriverError::ResponseDropped.into())+ }++ /// Returning this error from the host handler aborts [`drive`].+ pub fn fail(self, error: LibsyError) -> Result<()> {+ Err(error)+ }+}+/// The terminal result of routing.pub struct RoutingOutcome {/// Models selected by the algorithm, ordered best model first.@@ -213,7 +240,7 @@ impl RoutingOutcome {}}-/// How an algorithm's [`route`](Algorithm::route) makes model calls.+/// An algorithm's handle for requesting external work and awaiting the host's typed replies.#[derive(Clone)]pub struct Driver {step_tx: mpsc::Sender<Result<Step>>,@@ -332,6 +359,35 @@ impl Driver {result}+ /// Override the request's model with `model` and wait for the host to reply.+ #[tracing::instrument(+ target = "libsy",+ name = "libsy.decision_call",+ skip_all,+ fields(algorithm = self.algorithm, selected_model = %model),+ )]+ pub async fn call_decision(+ &self,+ mut request: DecisionRequest,+ model: ModelId,+ ) -> Result<DecisionResponse> {+ request.model = Some(model.clone());+ let (reply, response) = oneshot::channel();+ let call = CallDecision {+ algorithm: self.algorithm.clone(),+ request,+ model,+ reply,+ };+ self.step_tx+ .send(Ok(Step::CallDecision(Box::new(call))))+ .await+ .map_err(|_| DriverError::StreamClosed)?;+ response+ .await+ .map_err(|_| LibsyError::from(DriverError::ResponseDropped))?+ }+/// The available models for this category, typically ordered best-first.pub fn models_for(&self, category: &Category) -> &[ModelId] {match self.scope {@@ -390,28 +446,36 @@ impl Driver {}}-/// One item in the stream returned by [`Algorithm::run_stream`].+/// Work requests and the terminal outcome of [`Algorithm::run_stream`].+/// Each call carries its own reply channel, so the host can execute calls concurrently.pub enum Step {/// The algorithm needs this model call performed. The host serves it and fulfills/// it with [`CallModel::respond`]. Boxed: it is by far the largest variant.CallModel(Box<CallModel>),+ /// The host fulfills this decision call with [`CallDecision::respond`].+ CallDecision(Box<CallDecision>),/// The algorithm finished with its routing outcome — the last step of a run.Done(Box<RoutingOutcome>),}-/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.+/// Work dispatched through one host handler, with a typed reply channel per variant.+/// [`drive`] handles terminal outcomes separately; the handler only executes calls.+pub enum Call {+ /// Fulfilled with an LLM [`Response`].+ Model(Box<CallModel>),+ /// Fulfilled with a [`DecisionResponse`].+ Decision(Box<CallDecision>),+}++/// Drive [`Algorithm::run_stream`] to its routing outcome, serving calls concurrently.+///+/// The shared execution loop lets hosts supply clients or test doubles through `serve`+/// without reimplementing stream handling and cancellation.///-/// Returns the final [`RoutingOutcome`].-/// `serve` owns the call: it performs it however the host likes and must fulfill the promise-/// with [`CallModel::respond`]. A failed *model* call belongs in `respond` — the-/// algorithm may route around it. To stop routing on a model-call failure, return-/// [`CallModel::fail`] instead. Returning `Err` from `serve` aborts the whole run.-/// Calls are served concurrently, so an algorithm that offloads several at once (hedging, fan-out)-/// gets real parallelism.+/// `serve` owns each [`Call`] and performs its I/O. Passing an error to `respond`+/// lets the algorithm fall back; returning an error from `serve` aborts the run.///-/// libsy performs no I/O; this is only the mechanics of consuming its own step stream, kept-/// here so every host does not reimplement the same loop. `switchyard-llm-client`'s `run`-/// is this function plus an HTTP client.+/// Dropping this future cancels the algorithm and pending handler futures.pub async fn drive<F, Fut>(algorithm: Arc<dyn Algorithm>,request: Request,@@ -419,7 +483,7 @@ pub async fn drive<F, Fut>(serve: F,) -> Result<RoutingOutcome>where- F: Fn(CallModel) -> Fut,+ F: Fn(Call) -> Fut,Fut: Future<Output = Result<()>>,{let stream = algorithm.run_stream(request, models);@@ -431,14 +495,17 @@ whereloop {tokio::select! {Some(result) = in_flight.next() => match result {- Ok(()) => {}, // CallModel completed successfully- Err(err) => return Err(err), // CallModel failed, propagate the error+ Ok(()) => {}, // Call completed successfully+ Err(err) => return Err(err), // Call failed, propagate the error},step = stream.next() => {match step {None => break, // stream has ended, no more stepsSome(item) => match item? {- Step::CallModel(call) => in_flight.push(serve(*call)),+ Step::CallModel(call) => in_flight.push(serve(Call::Model(call))),+ Step::CallDecision(call) => {+ in_flight.push(serve(Call::Decision(call)));+ }Step::Done(outcome) => {final_outcome = Some(*outcome);break;@@ -513,10 +580,10 @@ impl RoutingIdentity {}}-/// An optimization strategy. Implement [`route`](Self::route);-/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallModel`]-/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this-/// over HTTP.+/// Routing policy independent of how external calls are executed.+/// Implement [`route`](Self::route), using [`Driver`] to request work from the host.+/// Hosts can use [`drive`] for the shared execution loop or consume+/// [`run_stream`](Self::run_stream) for custom execution.////// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across/// requests and run concurrently, so it owns its thread-safety and any shared state.@@ -552,15 +619,13 @@ pub trait Algorithm: Send + Sync + 'static {/// emits for its runs.fn name(&self) -> &str;- /// Run one request to completion: make routing-time model calls with- /// [`Driver::call_model`] and return the terminal [`RoutingOutcome`].- /// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.+ /// Select a route, requesting external work through [`Driver`] as needed.+ /// [`run_stream`](Self::run_stream) runs this method and exposes its work to the host.async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome>;- /// Process a request to completion, returning a stream of [`Step`]s.+ /// Expose work requests and the final outcome so the host can control execution.///- /// The consumer must fulfill every [`Step::CallModel`] before the algorithm can- /// continue. Every run ends with exactly one terminal item — [`Step::Done`] on+ /// Each call waits for its host reply. A run ends with one terminal item — [`Step::Done`] on/// success, an `Err` item on failure, including when the algorithm panics. Dropping/// the stream aborts the spawned algorithm task.///@@ -604,7 +669,7 @@ mod tests {use std::collections::HashMap;use super::*;- use crate::core::testing::{Serve, ServeResult, echo, reply, test_drive};+ use crate::core::testing::{Serve, ServeResult, echo, reply, serve_decision, test_drive};use futures::StreamExt;use switchyard_protocol::{LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,@@ -798,6 +863,138 @@ mod tests {result,Err(LibsyError::Driver(DriverError::StreamClosed))));++ fn decision_request() -> DecisionRequest {+ DecisionRequest {+ model: Some("overwritten".into()),+ context: serde_json::json!({"task": "choose a route"}),+ questions: Default::default(),+ }+ }++ fn decision_response() -> DecisionResponse {+ DecisionResponse {+ id: Some("decision-1".to_string()),+ model: Some("provider-model".into()),+ answers: [(+ "p_solve".to_string(),+ switchyard_protocol::DecisionAnswer {+ value: switchyard_protocol::DecisionValue::Boolean(+ switchyard_protocol::BooleanEstimate::ProbabilityTrue(+ switchyard_protocol::Probability(0.8),+ ),+ ),+ provider_confidence: None,+ },+ )]+ .into(),+ usage: Default::default(),+ }+ }++ struct MixedCalls(&'static str);++ #[async_trait]+ impl Algorithm for MixedCalls {+ fn name(&self) -> &str {+ "mixed"+ }++ async fn route(+ self: Arc<Self>,+ driver: Driver,+ request: Request,+ ) -> Result<RoutingOutcome> {+ let (llm, decision) = tokio::join!(+ driver.call_model(request.clone(), vec!["llm".into()]),+ driver.call_decision(decision_request(), "decision".into()),+ );+ assert_eq!(+ llm?.llm_response.as_agg().map(completion_text),+ Some("llm reply".into())+ );+ match self.0 {+ "mock" => assert_eq!(decision?, DecisionResponse {+ id: None,+ model: Some("decision".into()),+ answers: Default::default(),+ usage: Default::default(),+ }),+ "reply" => assert_eq!(decision?, decision_response()),+ "error" => assert!(matches!(+ decision,+ Err(LibsyError::AlgorithmError { message }) if message == "provider failed"+ )),+ "drop" => assert!(matches!(+ decision,+ Err(LibsyError::Driver(DriverError::ResponseDropped))+ )),+ "abort" => return std::future::pending().await,+ _ => unreachable!(),+ }+ Ok(RoutingOutcome::route_to("answer".into(), vec![], request))+ }+ }++ for mode in ["mock", "reply", "error", "drop", "abort"] {+ // Serial dispatch would deadlock here and fail the enclosing timeout.+ let barrier = Arc::new(tokio::sync::Barrier::new(2));+ let outcome = drive(+ Arc::new(MixedCalls(mode)),+ request(),+ Arc::new(RuntimeModels::default()),+ move |call| {+ let barrier = barrier.clone();+ async move {+ barrier.wait().await;+ let call = match call {+ Call::Model(call) => return call.respond(Ok(reply("llm reply"))),+ Call::Decision(call) => *call,+ };+ assert_eq!(call.algorithm, "mixed");+ assert_eq!(call.model, "decision");+ assert_eq!(call.request.model, Some("decision".into()));+ assert_eq!(call.request.context, decision_request().context);+ match mode {+ "mock" => serve_decision(call).await,+ "reply" => call.respond(Ok(decision_response())),+ "error" => call.respond(Err(LibsyError::AlgorithmError {+ message: "provider failed".into(),+ })),+ "drop" => {+ drop(call);+ Ok(())+ }+ "abort" => call.fail(test_error("host aborted")),+ _ => unreachable!(),+ }+ }+ },+ )+ .await;+ if mode == "abort" {+ assert!(matches!(outcome, Err(LibsyError::External { .. })));+ } else {+ assert_eq!(outcome?.selected_model_id()?, "answer");+ }+ }++ let (driver, mut steps) = Driver::new("test", Arc::new(RuntimeModels::default()));+ let mut pending = Box::pin(driver.call_decision(decision_request(), "decision".into()));+ assert!(futures::poll!(&mut pending).is_pending());+ let Some(Ok(Step::CallDecision(call))) = steps.recv().await else {+ return Err(test_error("expected a decision call"));+ };+ drop(pending);+ assert!(matches!(+ call.respond(Ok(decision_response())),+ Err(LibsyError::Driver(DriverError::ResponseDropped))+ ));+ drop(steps);+ assert!(matches!(+ driver.call_decision(decision_request(), "decision".into()).await,+ Err(LibsyError::Driver(DriverError::StreamClosed))+ ));Ok(())}).await@@ -901,6 +1098,7 @@ mod tests {let mut final_completion = None;while let Some(step) = stream.next().await {match step? {+ Step::CallDecision(_) => return Err(test_error("unexpected decision call")),Step::CallModel(call) => {saw_call = true;assert_eq!(call.models, vec![ModelId::from("offload/model")]);@@ -1014,6 +1212,7 @@ mod tests {let mut saw_error = false;while let Some(step) = stream.next().await {match step {+ Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),Ok(Step::CallModel(call)) => {call.respond(Err(test_error("upstream model call failed")))?;}diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsindex 3cc3a782d..b62667cb3 100644--- a/crates/libsy/src/core/testing.rs+++ b/crates/libsy/src/core/testing.rs@@ -19,10 +19,11 @@ use std::sync::Arc;use futures::future::BoxFuture;use switchyard_protocol::{- Category, LlmClientError, LlmResponse, ModelId, Request, Response, text_response,+ Category, DecisionResponse, LlmClientError, LlmResponse, ModelId, Request, Response,+ text_response,};-use crate::core::algorithm::{Algorithm, CallModel, Driver, RuntimeModels};+use crate::core::algorithm::{Algorithm, Call, CallDecision, Driver, RuntimeModels};use crate::{LibsyError, Result};/// Builds one runtime model category for a test.@@ -93,9 +94,23 @@ pub(crate) async fn test_drive_with_models(Ok((selected_model, response))}+pub(crate) async fn serve_decision(call: CallDecision) -> Result<()> {+ let response = DecisionResponse {+ id: None,+ model: Some(call.model.clone()),+ answers: Default::default(),+ usage: Default::default(),+ };+ call.respond(Ok(response))+}+/// Serve one call and fulfill its promise, mapping failures the way a host does so/// error-shape assertions match production.-async fn fulfill(serve: Arc<impl Serve>, call: CallModel) -> Result<()> {+async fn fulfill(serve: Arc<impl Serve>, call: Call) -> Result<()> {+ let call = match call {+ Call::Model(call) => *call,+ Call::Decision(call) => return serve_decision(*call).await,+ };let request = call.request.clone();let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;let result = servediff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex a1f1ef022..9b484ae27 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -6,7 +6,8 @@mod core;pub use core::algorithm::{- Algorithm, CallModel, Driver, RoutingOutcome, RuntimeModels, Step, StepStream, drive,+ Algorithm, Call, CallDecision, CallModel, Driver, RoutingOutcome, RuntimeModels, Step,+ StepStream, drive,};pub use core::classifier::{Classification, Classifier, Score};pub use core::outcome_metadata::OutcomeMetadata;diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rsindex 0d200ba28..bd66d230d 100644--- a/crates/prefill-router/tests/unit/algorithm.rs+++ b/crates/prefill-router/tests/unit/algorithm.rs@@ -4,7 +4,7 @@use std::sync::Arc;use std::sync::atomic::{AtomicUsize, Ordering};-use libsy::{Algorithm, LibsyError, RuntimeModels};+use libsy::{Algorithm, Call, LibsyError, RuntimeModels};use switchyard_protocol::{Category, ContentBlock, LlmRequest, Message, ModelId, Request, Role, ToolResult, text_request,};@@ -65,12 +65,26 @@ fn forward() -> ()}+async fn serve_decision(call: libsy::CallDecision) -> libsy::Result<()> {+ let response = switchyard_protocol::DecisionResponse {+ id: None,+ model: Some(call.model.clone()),+ answers: Default::default(),+ usage: Default::default(),+ };+ call.respond(Ok(response))+}+async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<String> {let outcome = libsy::drive(route,request,Arc::new(RuntimeModels::new([(Category::Any, target_set())].into())),|call| async move {+ let call = match call {+ Call::Model(call) => *call,+ Call::Decision(call) => return serve_decision(*call).await,+ };call.respond(Ok(switchyard_protocol::Response {llm_response: switchyard_protocol::LlmResponse::Agg(switchyard_protocol::text_response(None, "unused"),diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex e0d86d365..64484ccd8 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -9,7 +9,9 @@ use std::sync::Arc;use futures::StreamExt;use http::HeaderMap;use http::header::{HeaderName, HeaderValue};-use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError};+use pyo3::exceptions::{+ PyBaseException, PyNotImplementedError, PyStopAsyncIteration, PyTypeError, PyValueError,+};use pyo3::prelude::*;use serde_json::Value;use switchyard_libsy::{@@ -764,6 +766,9 @@ impl PyAlgorithm {fn step_to_python(step: RustStep) -> PyResult<PyStep> {match step {+ RustStep::CallDecision(_) => Err(PyNotImplementedError::new_err(+ "decision calls are not supported by the Python bindings",+ )),RustStep::CallModel(call) => Python::attach(|py| {Ok(PyStep::CallModel {call: Py::new(py, PyModelCall::new(py, *call)?)?,