Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: Rename ReturnToAgent to Done, create_run_task to route (#361)

Graham King1e5767b82bd8
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index a457ba996..18d8f9f35 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -437,7 +437,7 @@ impl Algorithm for SingleCallAlgo {
&self.name
}
- async fn create_run_task(
+ async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
@@ -1097,7 +1097,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
call.respond(Err(test_error("synthetic upstream failure")))?;
}
Ok(Step::Decision(_)) => {}
- Ok(Step::ReturnToAgent(_)) => {
+ Ok(Step::Done(_)) => {
return Err(test_error("expected the failed call to fail the run"));
}
Err(_) => saw_error_step = true,
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 2f172560a..f1012fb04 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -424,11 +424,7 @@ where
&self.name
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
self.execute(driver, request).await
}
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index be0536df7..80c173a87 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -930,11 +930,7 @@ impl Algorithm for LlmTaskClassifier {
"llm_task_classifier"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
self.route.execute(driver, request).await
}
}
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 64810991b..a14176b28 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -22,11 +22,7 @@ impl Algorithm for Noop {
"noop"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let model = request
.requested_model()
.unwrap_or("switchyard/noop")
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index f2d9f7f64..1c7e322b1 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -29,11 +29,7 @@ impl Algorithm for Passthrough {
"passthrough"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let decision: Decision = Decision::new(
self.target.semantic_name.clone(),
Some(format!(
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 96f3bbed8..2a7b61c84 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -159,11 +159,7 @@ impl Algorithm for Random {
"random"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
self.inner.execute(driver, request).await
}
}
@@ -178,7 +174,7 @@ mod tests {
use crate::algorithms::util::affinity::AffinityRouter;
use crate::core::algorithm::LlmTarget;
use crate::core::testing::{echo, test_drive};
- use switchyard_protocol::{Request, Signals};
+ use switchyard_protocol::Request;
fn request() -> Request {
Request {
@@ -402,13 +398,6 @@ mod tests {
assert!(error.contains("random targets must be unique"));
}
- #[tokio::test]
- async fn process_signals_is_a_noop() -> Result<()> {
- let algorithm: Arc<dyn Algorithm> = Arc::new(algorithm(&["only/model"], None, None)?);
- algorithm.process_signals(Signals {}).await?;
- Ok(())
- }
-
#[tokio::test]
async fn decision_is_inspectable() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index f90611e8c..c2ee63c39 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -143,11 +143,7 @@ impl Algorithm for StageRouter {
STAGE_ROUTER
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
self.route.execute(driver, request).await
}
}
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 34a33fee1..519b1e4bb 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -27,9 +27,7 @@ use tracing::Instrument;
/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and
/// [`switchyard_protocol::LlmResponse`] carries either a live
/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
-use switchyard_protocol::{
- Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,
-};
+use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason};
use crate::{DriverError, LibsyError, Result, observability};
@@ -55,10 +53,13 @@ pub struct CallModel {
/// The name of the algorithm that produced this call, so a host instrumenting the
/// calls it serves can attribute its own spans to the algorithm behind them.
pub algorithm: String,
- /// The request to serve; its `model` is the agent's original name.
+ /// The request to serve; its `model` is the agent's original name NOT the selected model.
+ /// The caller making the request needs to change it to decision.selected_model_id() before
+ /// sending.
pub request: Request,
- /// The routing decision behind this call; `selected_model_id()` identifies the model to hit.
+ /// The routing decision behind this call; `selected_model_id()` identifies the model to use.
pub decision: Decision,
+ // How to send the response back to the algorithm
reply: oneshot::Sender<Result<Response>>,
}
@@ -97,12 +98,7 @@ impl CallModel {
}
}
-/// The offload channel handed to an algorithm's
-/// [`create_run_task`](Algorithm::create_run_task). The algorithm makes model calls
-/// with [`call_model`](Self::call_model) and publishes its [`Decision`]s with
-/// [`info`](Self::info); each call is offloaded to the request's [`Step`] stream and
-/// awaits the consumer's response. The step channel is bounded, so the consumer paces
-/// the algorithm one step at a time.
+/// How an algorithm's [`route`](Algorithm::route) makes model calls.
#[derive(Clone)]
pub struct Driver {
step_tx: mpsc::Sender<Result<Step>>,
@@ -197,11 +193,11 @@ impl Driver {
Ok(())
}
- /// Emit the terminal step: [`Step::ReturnToAgent`] on `Ok`, or an `Err` stream
+ /// Emit the terminal step: [`Step::Done`] on `Ok`, or an `Err` stream
/// item on failure. Internal: called once by [`run_stream`](Algorithm::run_stream)
/// when the algorithm finishes.
pub(crate) async fn finish(&self, result: Result<Response>) -> Result<()> {
- let step = result.map(|response| Step::ReturnToAgent(Box::new(response)));
+ let step = result.map(|response| Step::Done(Box::new(response)));
self.step_tx
.send(step)
.await
@@ -218,7 +214,7 @@ pub enum Step {
/// happens (rather than collected into a trace returned at the end).
Decision(Decision),
/// The algorithm finished with its final response — the last step of a run.
- ReturnToAgent(Box<Response>),
+ Done(Box<Response>),
}
/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.
@@ -261,7 +257,7 @@ where
Some(item) => match item? {
Step::CallModel(call) => in_flight.push(serve(*call)),
Step::Decision(decision) => trace.push(decision),
- Step::ReturnToAgent(response) => {
+ Step::Done(response) => {
final_response = Some(*response);
break;
}
@@ -521,7 +517,7 @@ pub(crate) async fn call_model_with_fallback(
}
}
-/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
+/// An optimization strategy. Implement [`route`](Self::route);
/// callers drive it with [`run_stream`](Self::run_stream), serving each [`Step::CallModel`]
/// it emits. `switchyard-llm-client`'s `run` is the ready-made consumer that does this
/// over HTTP.
@@ -552,22 +548,13 @@ pub trait Algorithm: Send + Sync + 'static {
/// Run one request to completion: make model calls with [`Driver::call_model`],
/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
/// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.
- async fn create_run_task(self: Arc<Self>, driver: Driver, request: Request)
- -> Result<Response>;
-
- /// Feed the algorithm agentic-stack events (tool results, budgets, etc.). The
- /// reference algorithms ignore signals; a stateful algorithm updates its own
- /// (interior-mutable) state. Takes `self: Arc<Self>` like the other run methods.
- #[allow(unused_variables)]
- async fn process_signals(self: Arc<Self>, signals: Signals) -> Result<()> {
- Ok(())
- }
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response>;
/// Process a request to completion, returning a stream of [`Step`]s.
///
/// The consumer must fulfill every [`Step::CallModel`] before the algorithm can
/// continue. The bounded step channel applies backpressure when the consumer is
- /// not polling. A successful run ends with [`Step::ReturnToAgent`]; a failure is
+ /// not polling. A successful run ends with [`Step::Done`]; a failure is
/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
///
/// Every invocation owns a separate [`Driver`].
@@ -582,8 +569,7 @@ pub trait Algorithm: Send + Sync + 'static {
let handle = tokio::spawn(
async move {
let algorithm = self.name().to_string();
- observability::observe_run(&algorithm, self.create_run_task(task_driver, request))
- .await
+ observability::observe_run(&algorithm, self.route(task_driver, request)).await
}
.instrument(span),
);
@@ -700,11 +686,7 @@ mod tests {
"test"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let target = self
.target_set
.targets()
@@ -904,7 +886,7 @@ mod tests {
#[tokio::test]
async fn run_returns_a_streamed_response_the_caller_aggregates() -> Result<()> {
- // A streaming client -> its chunks flow through the promise and `ReturnToAgent`,
+ // A streaming client -> its chunks flow through the promise and `Done`,
// and `run` returns the live stream untouched for the caller to fold.
let (orch, serve) = streaming_orch(vec![
LlmResponseChunk::MessageStart {
@@ -960,7 +942,7 @@ mod tests {
}
#[tokio::test]
- async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
+ async fn run_offloads_via_promise_then_finishes() -> Result<()> {
// Every call is offloaded via a promise the orchestrator surfaces as a
// `CallModel` step for us to fulfill.
let stream = orch(target_set(&["offload/model"])).run_stream(request());
@@ -986,7 +968,7 @@ mod tests {
Step::Decision(decision) => {
assert_eq!(decision.selected_model_id(), "offload/model");
}
- Step::ReturnToAgent(response) => {
+ Step::Done(response) => {
final_completion = Some(
response
.llm_response
@@ -998,9 +980,9 @@ mod tests {
}
}
- assert!(saw_call, "expected a CallModel step before ReturnToAgent");
+ assert!(saw_call, "expected a CallModel step before Done");
assert_eq!(
- final_completion.ok_or_else(|| test_error("no ReturnToAgent step"))?,
+ final_completion.ok_or_else(|| test_error("no Done step"))?,
"fulfilled"
);
Ok(())
@@ -1088,7 +1070,7 @@ mod tests {
call.respond(Err(test_error("upstream model call failed")))?;
}
Ok(Step::Decision(_)) => {}
- Ok(Step::ReturnToAgent(..)) => {
+ Ok(Step::Done(..)) => {
return Err(test_error(
"expected the offload error to propagate, got a response",
));
@@ -1131,7 +1113,7 @@ mod tests {
"stuck"
}
- async fn create_run_task(
+ async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
@@ -1167,7 +1149,7 @@ mod tests {
}
#[tokio::test]
- async fn create_run_task_panic_surfaces_as_a_stream_error() -> Result<()> {
+ async fn route_panic_surfaces_as_a_stream_error() -> Result<()> {
// An algorithm whose task panics must surface an `Err` step to the stream
// consumer, not abort the process from an unobserved detached task.
struct Panicky;
@@ -1178,7 +1160,7 @@ mod tests {
"panicky"
}
- async fn create_run_task(
+ async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
@@ -1218,7 +1200,7 @@ mod tests {
"panicky"
}
- async fn create_run_task(
+ async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
@@ -1265,7 +1247,7 @@ mod tests {
"stuck"
}
- async fn create_run_task(
+ async fn route(
self: Arc<Self>,
_driver: Driver,
_request: Request,
@@ -1318,11 +1300,7 @@ mod tests {
"hedge"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let dec_w = test_decision(self.winner.semantic_name.clone());
let dec_l = test_decision(self.loser.semantic_name.clone());
let win = driver.call_model(request.clone(), dec_w);
@@ -1424,11 +1402,7 @@ mod tests {
"fan_out_then_error"
}
- async fn create_run_task(
- self: Arc<Self>,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let offloads = futures::future::join_all((0..self.n).map(|i| {
let decision = test_decision(format!("m{i}"));
driver.call_model(request.clone(), decision)
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index 8560402ba..7f23e7d0d 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -3,7 +3,7 @@
use crate::Result;
use async_trait::async_trait;
-use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};
+use switchyard_protocol::{AggLlmResponse, Decision, Request};
/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state.
///
@@ -13,8 +13,6 @@ use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};
pub enum Event<'a> {
/// The inbound request that begins a turn.
Request(&'a mut Request),
- /// An out-of-band agentic-stack signal (tool results, budget updates, …).
- Signal(&'a Signals),
/// A routing decision paired with the request that produced it.
///
/// The request is rewritable: a processor may add instructions or notes here that
@@ -49,7 +47,6 @@ mod tests {
fn event_key(event: &Event<'_>) -> &'static str {
match event {
Event::Request(_) => "requests",
- Event::Signal(_) => "signals",
Event::Decision { .. } => "decisions",
Event::ModelResponse(_) => "model_responses",
}
@@ -86,8 +83,6 @@ mod tests {
let mut req = request();
let response = text_response(None, "ok");
let decision = Decision::new("test/model", None, true);
- let signals = Signals {};
-
// Feed one of every event variant through the processor.
processor
.process(&mut state, Event::Request(&mut req))
@@ -104,12 +99,7 @@ mod tests {
},
)
.await?;
- processor
- .process(&mut state, Event::Signal(&signals))
- .await?;
-
assert_eq!(count(&state, "requests"), 1);
- assert_eq!(count(&state, "signals"), 1);
assert_eq!(count(&state, "decisions"), 1);
assert_eq!(count(&state, "model_responses"), 1);
Ok(())
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 56e55ed32..ca093dae5 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -90,7 +90,7 @@ pub(crate) fn outcome_value<T>(result: &Result<T>) -> &'static str {
}
}
-/// Span covering one algorithm run (the whole `create_run_task` execution).
+/// Span covering one algorithm run (the whole `route` execution).
///
/// Correlation ids from the request [`switchyard_protocol::Metadata`] are recorded as span fields
/// when present. `tracing` spans cannot grow field names at runtime, so
diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rs
index 0986dca16..227a9d4c6 100644
--- a/crates/protocol/src/envelope.rs
+++ b/crates/protocol/src/envelope.rs
@@ -6,14 +6,6 @@
use crate::{LlmRequest, LlmResponse, Metadata};
-/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget
-/// updates) — in libsy, via `Algorithm::process_signals`.
-///
-/// A placeholder today; a stateful algorithm can begin consuming signals as the enum
-/// grows without changing the orchestrator contract.
-#[derive(Clone)]
-pub struct Signals {}
-
/// A request an algorithm routes: the normalized [`LlmRequest`] plus optional
/// host-owned raw data and correlation [`Metadata`].
#[derive(Clone, Default)]
diff --git a/docs/getting_started.md b/docs/getting_started.md
index 74207065f..27bbd1265 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -264,7 +264,7 @@ behaviour.
An algorithm yields a stream of steps. Each `Step::CallModel` is a model call your
host performs over its own transport, and the run ends with
-`Step::ReturnToAgent` carrying the final response. Serving those calls yourself
+`Step::Done` carrying the final response. Serving those calls yourself
is what lets libsy embed in a host that already owns its HTTP stack, retries,
and credentials.