Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore: Rename ReturnToAgent to Done, create_run_task to route (#361)
Graham King1e5767b82bd8
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex a457ba996..18d8f9f35 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -437,7 +437,7 @@ impl Algorithm for SingleCallAlgo {&self.name}- async fn create_run_task(+ async fn route(self: Arc<Self>,driver: Driver,request: Request,@@ -1097,7 +1097,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::call.respond(Err(test_error("synthetic upstream failure")))?;}Ok(Step::Decision(_)) => {}- Ok(Step::ReturnToAgent(_)) => {+ Ok(Step::Done(_)) => {return Err(test_error("expected the failed call to fail the run"));}Err(_) => saw_error_step = true,diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 2f172560a..f1012fb04 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -424,11 +424,7 @@ where&self.name}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {self.execute(driver, request).await}}diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex be0536df7..80c173a87 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -930,11 +930,7 @@ impl Algorithm for LlmTaskClassifier {"llm_task_classifier"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {self.route.execute(driver, request).await}}diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex 64810991b..a14176b28 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -22,11 +22,7 @@ impl Algorithm for Noop {"noop"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let model = request.requested_model().unwrap_or("switchyard/noop")diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex f2d9f7f64..1c7e322b1 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -29,11 +29,7 @@ impl Algorithm for Passthrough {"passthrough"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let decision: Decision = Decision::new(self.target.semantic_name.clone(),Some(format!(diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex 96f3bbed8..2a7b61c84 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -159,11 +159,7 @@ impl Algorithm for Random {"random"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {self.inner.execute(driver, request).await}}@@ -178,7 +174,7 @@ mod tests {use crate::algorithms::util::affinity::AffinityRouter;use crate::core::algorithm::LlmTarget;use crate::core::testing::{echo, test_drive};- use switchyard_protocol::{Request, Signals};+ use switchyard_protocol::Request;fn request() -> Request {Request {@@ -402,13 +398,6 @@ mod tests {assert!(error.contains("random targets must be unique"));}- #[tokio::test]- async fn process_signals_is_a_noop() -> Result<()> {- let algorithm: Arc<dyn Algorithm> = Arc::new(algorithm(&["only/model"], None, None)?);- algorithm.process_signals(Signals {}).await?;- Ok(())- }-#[tokio::test]async fn decision_is_inspectable() -> Result<()> {let algorithm = shared_algorithm(&["only/model"])?;diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex f90611e8c..c2ee63c39 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -143,11 +143,7 @@ impl Algorithm for StageRouter {STAGE_ROUTER}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {self.route.execute(driver, request).await}}diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 34a33fee1..519b1e4bb 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -27,9 +27,7 @@ use tracing::Instrument;/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and/// [`switchyard_protocol::LlmResponse`] carries either a live/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.-use switchyard_protocol::{- Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,-};+use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason};use crate::{DriverError, LibsyError, Result, observability};@@ -55,10 +53,13 @@ pub struct CallModel {/// The name of the algorithm that produced this call, so a host instrumenting the/// calls it serves can attribute its own spans to the algorithm behind them.pub algorithm: String,- /// The request to serve; its `model` is the agent's original name.+ /// The request to serve; its `model` is the agent's original name NOT the selected model.+ /// The caller making the request needs to change it to decision.selected_model_id() before+ /// sending.pub request: Request,- /// The routing decision behind this call; `selected_model_id()` identifies the model to hit.+ /// The routing decision behind this call; `selected_model_id()` identifies the model to use.pub decision: Decision,+ // How to send the response back to the algorithmreply: oneshot::Sender<Result<Response>>,}@@ -97,12 +98,7 @@ impl CallModel {}}-/// The offload channel handed to an algorithm's-/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls-/// with [`call_model`](Self::call_model) and publishes its [`Decision`]s with-/// [`info`](Self::info); each call is offloaded to the request's [`Step`] stream and-/// awaits the consumer's response. The step channel is bounded, so the consumer paces-/// the algorithm one step at a time.+/// How an algorithm's [`route`](Algorithm::route) makes model calls.#[derive(Clone)]pub struct Driver {step_tx: mpsc::Sender<Result<Step>>,@@ -197,11 +193,11 @@ impl Driver {Ok(())}- /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream+ /// Emit the terminal step: [`Step::Done`] on `Ok`, or an `Err` stream/// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)/// when the algorithm finishes.pub(crate) async fn finish(&self, result: Result<Response>) -> Result<()> {- let step = result.map(|response| Step::ReturnToAgent(Box::new(response)));+ let step = result.map(|response| Step::Done(Box::new(response)));self.step_tx.send(step).await@@ -218,7 +214,7 @@ pub enum Step {/// happens (rather than collected into a trace returned at the end).Decision(Decision),/// The algorithm finished with its final response — the last step of a run.- ReturnToAgent(Box<Response>),+ Done(Box<Response>),}/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.@@ -261,7 +257,7 @@ whereSome(item) => match item? {Step::CallModel(call) => in_flight.push(serve(*call)),Step::Decision(decision) => trace.push(decision),- Step::ReturnToAgent(response) => {+ Step::Done(response) => {final_response = Some(*response);break;}@@ -521,7 +517,7 @@ pub(crate) async fn call_model_with_fallback(}}-/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);+/// An optimization strategy. Implement [`route`](Self::route);/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallModel`]/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this/// over HTTP.@@ -552,22 +548,13 @@ pub trait Algorithm: Send + Sync + 'static {/// Run one request to completion: make model calls with [`Driver::call_model`],/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`]./// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.- async fn create_run_task(self: Arc<Self>, driver: Driver, request: Request)- -> Result<Response>;-- /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The- /// reference algorithms ignore signals; a stateful algorithm updates its own- /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.- #[allow(unused_variables)]- async fn process_signals(self: Arc<Self>, signals: Signals) -> Result<()> {- Ok(())- }+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response>;/// Process a request to completion, returning a stream of [`Step`]s.////// The consumer must fulfill every [`Step::CallModel`] before the algorithm can/// continue. The bounded step channel applies backpressure when the consumer is- /// not polling. A successful run ends with [`Step::ReturnToAgent`]; a failure is+ /// not polling. A successful run ends with [`Step::Done`]; a failure is/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.////// Every invocation owns a separate [`Driver`].@@ -582,8 +569,7 @@ pub trait Algorithm: Send + Sync + 'static {let handle = tokio::spawn(async move {let algorithm = self.name().to_string();- observability::observe_run(&algorithm, self.create_run_task(task_driver, request))- .await+ observability::observe_run(&algorithm, self.route(task_driver, request)).await}.instrument(span),);@@ -700,11 +686,7 @@ mod tests {"test"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let target = self.target_set.targets()@@ -904,7 +886,7 @@ mod tests {#[tokio::test]async fn run_returns_a_streamed_response_the_caller_aggregates() -> Result<()> {- // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,+ // A streaming client -> its chunks flow through the promise and `Done`,// and `run` returns the live stream untouched for the caller to fold.let (orch, serve) = streaming_orch(vec![LlmResponseChunk::MessageStart {@@ -960,7 +942,7 @@ mod tests {}#[tokio::test]- async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {+ async fn run_offloads_via_promise_then_finishes() -> Result<()> {// Every call is offloaded via a promise the orchestrator surfaces as a// `CallModel` step for us to fulfill.let stream = orch(target_set(&["offload/model"])).run_stream(request());@@ -986,7 +968,7 @@ mod tests {Step::Decision(decision) => {assert_eq!(decision.selected_model_id(), "offload/model");}- Step::ReturnToAgent(response) => {+ Step::Done(response) => {final_completion = Some(response.llm_response@@ -998,9 +980,9 @@ mod tests {}}- assert!(saw_call, "expected a CallModel step before ReturnToAgent");+ assert!(saw_call, "expected a CallModel step before Done");assert_eq!(- final_completion.ok_or_else(|| test_error("no ReturnToAgent step"))?,+ final_completion.ok_or_else(|| test_error("no Done step"))?,"fulfilled");Ok(())@@ -1088,7 +1070,7 @@ mod tests {call.respond(Err(test_error("upstream model call failed")))?;}Ok(Step::Decision(_)) => {}- Ok(Step::ReturnToAgent(..)) => {+ Ok(Step::Done(..)) => {return Err(test_error("expected the offload error to propagate, got a response",));@@ -1131,7 +1113,7 @@ mod tests {"stuck"}- async fn create_run_task(+ async fn route(self: Arc<Self>,_driver: Driver,_request: Request,@@ -1167,7 +1149,7 @@ mod tests {}#[tokio::test]- async fn create_run_task_panic_surfaces_as_a_stream_error() -> Result<()> {+ async fn route_panic_surfaces_as_a_stream_error() -> Result<()> {// An algorithm whose task panics must surface an `Err` step to the stream// consumer, not abort the process from an unobserved detached task.struct Panicky;@@ -1178,7 +1160,7 @@ mod tests {"panicky"}- async fn create_run_task(+ async fn route(self: Arc<Self>,_driver: Driver,_request: Request,@@ -1218,7 +1200,7 @@ mod tests {"panicky"}- async fn create_run_task(+ async fn route(self: Arc<Self>,_driver: Driver,_request: Request,@@ -1265,7 +1247,7 @@ mod tests {"stuck"}- async fn create_run_task(+ async fn route(self: Arc<Self>,_driver: Driver,_request: Request,@@ -1318,11 +1300,7 @@ mod tests {"hedge"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let dec_w = test_decision(self.winner.semantic_name.clone());let dec_l = test_decision(self.loser.semantic_name.clone());let win = driver.call_model(request.clone(), dec_w);@@ -1424,11 +1402,7 @@ mod tests {"fan_out_then_error"}- async fn create_run_task(- self: Arc<Self>,- driver: Driver,- request: Request,- ) -> Result<Response> {+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let offloads = futures::future::join_all((0..self.n).map(|i| {let decision = test_decision(format!("m{i}"));driver.call_model(request.clone(), decision)diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rsindex 8560402ba..7f23e7d0d 100644--- a/crates/libsy/src/core/processor.rs+++ b/crates/libsy/src/core/processor.rs@@ -3,7 +3,7 @@use crate::Result;use async_trait::async_trait;-use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};+use switchyard_protocol::{AggLlmResponse, Decision, Request};/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state.///@@ -13,8 +13,6 @@ use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};pub enum Event<'a> {/// The inbound request that begins a turn.Request(&'a mut Request),- /// An out-of-band agentic-stack signal (tool results, budget updates, …).- Signal(&'a Signals),/// A routing decision paired with the request that produced it.////// The request is rewritable: a processor may add instructions or notes here that@@ -49,7 +47,6 @@ mod tests {fn event_key(event: &Event<'_>) -> &'static str {match event {Event::Request(_) => "requests",- Event::Signal(_) => "signals",Event::Decision { .. } => "decisions",Event::ModelResponse(_) => "model_responses",}@@ -86,8 +83,6 @@ mod tests {let mut req = request();let response = text_response(None, "ok");let decision = Decision::new("test/model", None, true);- let signals = Signals {};-// Feed one of every event variant through the processor.processor.process(&mut state, Event::Request(&mut req))@@ -104,12 +99,7 @@ mod tests {},).await?;- processor- .process(&mut state, Event::Signal(&signals))- .await?;-assert_eq!(count(&state, "requests"), 1);- assert_eq!(count(&state, "signals"), 1);assert_eq!(count(&state, "decisions"), 1);assert_eq!(count(&state, "model_responses"), 1);Ok(())diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 56e55ed32..ca093dae5 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -90,7 +90,7 @@ pub(crate) fn outcome_value<T>(result: &Result<T>) -> &'static str {}}-/// Span covering one algorithm run (the whole `create_run_task` execution).+/// Span covering one algorithm run (the whole `route` execution).////// Correlation ids from the request [`switchyard_protocol::Metadata`] are recorded as span fields/// when present. `tracing` spans cannot grow field names at runtime, sodiff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rsindex 0986dca16..227a9d4c6 100644--- a/crates/protocol/src/envelope.rs+++ b/crates/protocol/src/envelope.rs@@ -6,14 +6,6 @@use crate::{LlmRequest, LlmResponse, Metadata};-/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget-/// updates) — in libsy, via `Algorithm::process_signals`.-///-/// A placeholder today; a stateful algorithm can begin consuming signals as the enum-/// grows without changing the orchestrator contract.-#[derive(Clone)]-pub struct Signals {}-/// A request an algorithm routes: the normalized [`LlmRequest`] plus optional/// host-owned raw data and correlation [`Metadata`].#[derive(Clone, Default)]diff --git a/docs/getting_started.md b/docs/getting_started.mdindex 74207065f..27bbd1265 100644--- a/docs/getting_started.md+++ b/docs/getting_started.md@@ -264,7 +264,7 @@ behaviour.An algorithm yields a stream of steps. Each `Step::CallModel` is a model call yourhost performs over its own transport, and the run ends with-`Step::ReturnToAgent` carrying the final response. Serving those calls yourself+`Step::Done` carrying the final response. Serving those calls yourselfis what lets libsy embed in a host that already owns its HTTP stack, retries,and credentials.