Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: RoutingOutcome has a Vec of selected_model_ids (#592)
Graham King477ad1feb2b4
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 454ae5f30..8103a3bb6 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -67,13 +67,10 @@ pub async fn run(let overhead = run_started.elapsed();metrics::record_routing_overhead(&algorithm_name, overhead);- let selected_model_id = outcome.selected_model_id;+ let selected_model_id = outcome.selected_model_id()?.clone();let (result, answer_duration) = if let Some(response) = outcome.response {(Ok(response), None)} else {- let mut models = Vec::with_capacity(1 + outcome.fallback_models.len());- models.push(selected_model_id.clone());- models.extend(outcome.fallback_models);let answer_started = Instant::now();let observe = |observation| {if let Some(observer) = &observer {@@ -84,7 +81,7 @@ pub async fn run(&clients,&algorithm_name,&outcome.request,- &models,+ &outcome.selected_model_ids,CallPhase::Completion,&observe,)@@ -119,8 +116,8 @@ pub async fn decide(serve(routing_clients.clone(), call, None)}).await?;- outcome.request =- clients.prepare_completion_request(outcome.request, &outcome.selected_model_id);+ let selected_model_id = outcome.selected_model_id()?.clone();+ outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);Ok(outcome)}@@ -750,7 +747,10 @@ mod tests {).await?;- assert_eq!(outcome.fallback_models, [ModelId::from("strong")]);+ assert_eq!(+ outcome.selected_model_ids,+ [ModelId::from("weak"), ModelId::from("strong")]+ );assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);Ok(())}diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 7f6dd9eaf..2146d96fb 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -641,8 +641,10 @@ mod tests {tokio::pin!(stream);while let Some(step) = stream.next().await {if let crate::Step::Done(outcome) = step? {- assert_eq!(outcome.selected_model_id, ModelId::from("mid"));- assert_eq!(outcome.fallback_models, target_set(&["weak", "strong"]));+ assert_eq!(+ outcome.selected_model_ids,+ target_set(&["mid", "weak", "strong"])+ );assert_eq!(outcome.request.llm_request.model.as_deref(), Some("mid"));assert!(outcome.response.is_none());return Ok(());diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 47ddfb668..9c89a4553 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -62,10 +62,8 @@ impl CallModel {/// The terminal result of routing.pub struct RoutingOutcome {- /// The model selected by the algorithm and tried first by the client.- pub selected_model_id: ModelId,- /// Additional models the client may try in order after an eligible failure.- pub fallback_models: Vec<ModelId>,+ /// Models selected by the algorithm, ordered best model first.+ pub selected_model_ids: Vec<ModelId>,/// The request after all routing-time rewrites, stamped with the selected model.pub request: Request,/// A response produced while routing, or `None` when the client must make the answer call.@@ -73,6 +71,12 @@ pub struct RoutingOutcome {}impl RoutingOutcome {+ /// The model the algorithm recommends, the best model for this request.+ /// `LibsyError::NoTargets` if the algorithm selected no models, which should be impossible.+ pub fn selected_model_id(&self) -> Result<&ModelId> {+ self.selected_model_ids.first().ok_or(LibsyError::NoTargets)+ }+/// The decision is that client should send this `request`. The `selected_model_id`/// will be written into it by this function./// If that fails client should try the `fallback_models` in order.@@ -82,9 +86,11 @@ impl RoutingOutcome {mut request: Request,) -> Self {request.llm_request.model = Some(selected_model_id.to_string());+ let mut selected_model_ids = Vec::with_capacity(1 + fallback_models.len());+ selected_model_ids.push(selected_model_id);+ selected_model_ids.extend(fallback_models);Self {- selected_model_id,- fallback_models,+ selected_model_ids,request,response: None,}@@ -95,8 +101,7 @@ impl RoutingOutcome {pub fn answered(selected_model_id: ModelId, mut request: Request, response: Response) -> Self {request.llm_request.model = Some(selected_model_id.to_string());Self {- selected_model_id,- fallback_models: Vec::new(),+ selected_model_ids: vec![selected_model_id],request,response: Some(response),}@@ -194,7 +199,7 @@ impl Driver {let selected_model = result.as_ref().ok()- .map(|outcome| outcome.selected_model_id.clone());+ .and_then(|outcome| outcome.selected_model_id().ok().cloned());let step = result.map(|outcome| Step::Done(Box::new(outcome)));self.step_tx.send(step)@@ -472,16 +477,15 @@ mod tests {request(),);- assert_eq!(outcome.selected_model_id, "selected");assert_eq!(- outcome.fallback_models,- target_set(&["fallback-one", "fallback-two"])+ outcome.selected_model_ids,+ target_set(&["selected", "fallback-one", "fallback-two"]));assert_eq!(outcome.request.model_id().as_deref(), Some("selected"));assert!(outcome.response.is_none());let outcome = RoutingOutcome::route_to("only".into(), Vec::new(), request());- assert!(outcome.fallback_models.is_empty());+ assert_eq!(outcome.selected_model_ids, target_set(&["only"]));let outcome = RoutingOutcome::answered("answered".into(),@@ -492,9 +496,8 @@ mod tests {},);- assert_eq!(outcome.selected_model_id, "answered");+ assert_eq!(outcome.selected_model_ids, target_set(&["answered"]));assert_eq!(outcome.request.model_id().as_deref(), Some("answered"));- assert!(outcome.fallback_models.is_empty());assert_eq!(outcome.responsediff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsindex 3fdb8c38b..aeb570988 100644--- a/crates/libsy/src/core/testing.rs+++ b/crates/libsy/src/core/testing.rs@@ -53,7 +53,7 @@ pub(crate) async fn test_drive(fulfill(Arc::clone(&routing_serve), call)}).await?;- let selected_model = outcome.selected_model_id.clone();+ let selected_model = outcome.selected_model_id()?.clone();let response = match outcome.response {Some(response) => response,None => servediff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex e2d6a4eaf..0571a3e12 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -517,24 +517,17 @@ impl PyModelCall {/// The terminal routing selection, rewritten request, and optional existing response.#[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)]struct PyRoutingOutcome {- selected_model_id: String,- fallback_models: Vec<String>,+ selected_model_ids: Vec<String>,request: Py<PyAny>,response: Option<Py<PyAny>>,}#[pymethods]impl PyRoutingOutcome {- /// The model selected by the algorithm and tried first by the host.+ /// Models selected by the algorithm, ordered best model first.#[getter]- fn selected_model_id(&self) -> &str {- &self.selected_model_id- }-- /// Additional models the host may try in order after an eligible failure.- #[getter]- fn fallback_models(&self) -> Vec<String> {- self.fallback_models.clone()+ fn selected_model_ids(&self) -> Vec<String> {+ self.selected_model_ids.clone()}/// The normalized request after routing-time rewrites.@@ -679,8 +672,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {}),RustStep::Done(outcome) => {let RoutingOutcome {- selected_model_id,- fallback_models,+ selected_model_ids,request,response,} = *outcome;@@ -689,8 +681,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {outcome: Py::new(py,PyRoutingOutcome {- selected_model_id: selected_model_id.to_string(),- fallback_models: fallback_models+ selected_model_ids: selected_model_ids.iter().map(ToString::to_string).collect(),diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rsindex bcd175344..36ecb6410 100644--- a/crates/switchyard-runner/src/runner.rs+++ b/crates/switchyard-runner/src/runner.rs@@ -99,13 +99,10 @@ impl Runner {) -> Option<DecisionDescription> {let route = self.route(model.as_str())?;let resolve = |selected: &ModelId| route.decision_target(selected);+ let mut model_ids = outcome.selected_model_ids.iter();Some(DecisionDescription {- selected: resolve(&outcome.selected_model_id)?,- fallbacks: outcome- .fallback_models- .iter()- .map(resolve)- .collect::<Option<Vec<_>>>()?,+ selected: resolve(model_ids.next()?)?,+ fallbacks: model_ids.map(resolve).collect::<Option<Vec<_>>>()?,})}}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex eea1c7a2e..844931831 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -723,7 +723,7 @@ async fn decision(match encode_aggregated_response(&aggregate,input_format,- Some(outcome.selected_model_id.as_str()),+ outcome.selected_model_id().ok().map(ModelId::as_str),) {Ok(response) => Some(response),Err(error) => return server_error(error.to_string()),diff --git a/examples/libsy.py b/examples/libsy.pyindex 471ab5b55..fd6b8b684 100644--- a/examples/libsy.py+++ b/examples/libsy.py@@ -68,10 +68,10 @@ async def main() -> None:case Step.CallModel(call):call.respond(await client.call(call.request, call.models[0]))case Step.Done(outcome):- print("Decision:", outcome.selected_model_id)+ print("Decision:", outcome.selected_model_ids[0])response = outcome.response or await client.call(outcome.request,- outcome.selected_model_id,+ outcome.selected_model_ids[0],)match response:case LlmResponse.Agg(aggregate_response):diff --git a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.pyindex 1532204a1..333d9a80e 100644--- a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py+++ b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py@@ -181,7 +181,7 @@ class SwitchyardRoutingPlugin(LiteLLMRequestRewriter):"Switchyard algorithm produced a response while routing, ""which a LiteLLM routing plugin cannot return")- selected = outcome.selected_model_id+ selected = outcome.selected_model_ids[0]if selected not in candidates:raise ValueError(f"Switchyard selected {selected!r}, which is not in LiteLLM's "@@ -197,7 +197,7 @@ class SwitchyardRoutingPlugin(LiteLLMRequestRewriter):]context.signals["switchyard"] = {"selected_model_id": selected,- "fallback_models": outcome.fallback_models,+ "fallback_models": outcome.selected_model_ids[1:],}if request_patch["set"] or request_patch.get("remove"):context.signals["switchyard"]["request_patch"] = request_patchdiff --git a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py b/examples/litellm/tests/unit/test_switchyard_routing_plugin.pyindex 3038f0bb5..e0cca081c 100644--- a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py+++ b/examples/litellm/tests/unit/test_switchyard_routing_plugin.py@@ -166,15 +166,14 @@ async def test_litellm_conversion_preserves_stage_tool_signal_input() -> None:# ToolSignals reads only normalized messages, so exact equality protects every signal input.assert _request(litellm_messages)["messages"] == original_messagesassert direct_outcome is not None- assert direct_outcome.selected_model_id == SOL- assert direct_outcome.fallback_models == [TERRA]+ assert direct_outcome.selected_model_ids == [SOL, TERRA]context = routing_context(litellm_messages)await stage_plugin().run(context)assert context.signals["switchyard"] == {- "selected_model_id": direct_outcome.selected_model_id,- "fallback_models": direct_outcome.fallback_models,+ "selected_model_id": direct_outcome.selected_model_ids[0],+ "fallback_models": direct_outcome.selected_model_ids[1:],}diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex cb15172cc..1589d60ff 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -114,10 +114,7 @@ if TYPE_CHECKING:@finalclass RoutingOutcome:@property- def selected_model_id(self) -> str: ...-- @property- def fallback_models(self) -> list[str]: ...+ def selected_model_ids(self) -> list[str]: ...@propertydef request(self) -> dict[str, object]: ...diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex 07dd3cda7..02ac614e4 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -80,10 +80,10 @@ async def run_algorithm(if outcome.response is not None:match outcome.response:case LlmResponse.Agg(response):- return outcome.selected_model_id, response+ return outcome.selected_model_ids[0], responsecase LlmResponse.Stream(_):raise AssertionError("test helper expected an aggregate response")- candidates = [outcome.selected_model_id, *outcome.fallback_models]+ candidates = outcome.selected_model_idsfor index, target in enumerate(candidates):candidate_request = {**outcome.request, "model": target}client = (clients or {})[target]@@ -93,7 +93,7 @@ async def run_algorithm(if index + 1 == len(candidates):raiseelse:- return outcome.selected_model_id, response+ return outcome.selected_model_ids[0], responseraise AssertionError("algorithm stream ended without an outcome")@@ -111,8 +111,7 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()assert variants == ["done"]assert outcome is not None- assert outcome.selected_model_id == "fast"- assert outcome.fallback_models == []+ assert outcome.selected_model_ids == ["fast"]assert outcome.response is Noneresponse = await client.call(outcome.request)assert client.calls[0]["model"] == "fast"@@ -156,7 +155,7 @@ async def test_routing_call_accepts_a_streamed_response() -> None:outcome = doneassert outcome is not None- assert outcome.selected_model_id == "model-b"+ assert outcome.selected_model_ids == ["model-b", "model-a"]async def test_classifier_config_accepts_a_prompt_override() -> None: