Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: RoutingOutcome has a Vec of selected_model_ids (#592)

Graham King477ad1feb2b4
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 454ae5f30..8103a3bb6 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -67,13 +67,10 @@ pub async fn run(
let overhead = run_started.elapsed();
metrics::record_routing_overhead(&algorithm_name, overhead);
- let selected_model_id = outcome.selected_model_id;
+ let selected_model_id = outcome.selected_model_id()?.clone();
let (result, answer_duration) = if let Some(response) = outcome.response {
(Ok(response), None)
} else {
- let mut models = Vec::with_capacity(1 + outcome.fallback_models.len());
- models.push(selected_model_id.clone());
- models.extend(outcome.fallback_models);
let answer_started = Instant::now();
let observe = |observation| {
if let Some(observer) = &observer {
@@ -84,7 +81,7 @@ pub async fn run(
&clients,
&algorithm_name,
&outcome.request,
- &models,
+ &outcome.selected_model_ids,
CallPhase::Completion,
&observe,
)
@@ -119,8 +116,8 @@ pub async fn decide(
serve(routing_clients.clone(), call, None)
})
.await?;
- outcome.request =
- clients.prepare_completion_request(outcome.request, &outcome.selected_model_id);
+ let selected_model_id = outcome.selected_model_id()?.clone();
+ outcome.request = clients.prepare_completion_request(outcome.request, &selected_model_id);
Ok(outcome)
}
@@ -750,7 +747,10 @@ mod tests {
)
.await?;
- assert_eq!(outcome.fallback_models, [ModelId::from("strong")]);
+ assert_eq!(
+ outcome.selected_model_ids,
+ [ModelId::from("weak"), ModelId::from("strong")]
+ );
assert_eq!(instruction_text(&outcome.request), ["weak prompt"]);
Ok(())
}
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 7f6dd9eaf..2146d96fb 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -641,8 +641,10 @@ mod tests {
tokio::pin!(stream);
while let Some(step) = stream.next().await {
if let crate::Step::Done(outcome) = step? {
- assert_eq!(outcome.selected_model_id, ModelId::from("mid"));
- assert_eq!(outcome.fallback_models, target_set(&["weak", "strong"]));
+ assert_eq!(
+ outcome.selected_model_ids,
+ target_set(&["mid", "weak", "strong"])
+ );
assert_eq!(outcome.request.llm_request.model.as_deref(), Some("mid"));
assert!(outcome.response.is_none());
return Ok(());
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 47ddfb668..9c89a4553 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -62,10 +62,8 @@ impl CallModel {
/// The terminal result of routing.
pub struct RoutingOutcome {
- /// The model selected by the algorithm and tried first by the client.
- pub selected_model_id: ModelId,
- /// Additional models the client may try in order after an eligible failure.
- pub fallback_models: Vec<ModelId>,
+ /// Models selected by the algorithm, ordered best model first.
+ pub selected_model_ids: Vec<ModelId>,
/// The request after all routing-time rewrites, stamped with the selected model.
pub request: Request,
/// A response produced while routing, or `None` when the client must make the answer call.
@@ -73,6 +71,12 @@ pub struct RoutingOutcome {
}
impl RoutingOutcome {
+ /// The model the algorithm recommends, the best model for this request.
+ /// `LibsyError::NoTargets` if the algorithm selected no models, which should be impossible.
+ pub fn selected_model_id(&self) -> Result<&ModelId> {
+ self.selected_model_ids.first().ok_or(LibsyError::NoTargets)
+ }
+
/// The decision is that client should send this `request`. The `selected_model_id`
/// will be written into it by this function.
/// If that fails client should try the `fallback_models` in order.
@@ -82,9 +86,11 @@ impl RoutingOutcome {
mut request: Request,
) -> Self {
request.llm_request.model = Some(selected_model_id.to_string());
+ let mut selected_model_ids = Vec::with_capacity(1 + fallback_models.len());
+ selected_model_ids.push(selected_model_id);
+ selected_model_ids.extend(fallback_models);
Self {
- selected_model_id,
- fallback_models,
+ selected_model_ids,
request,
response: None,
}
@@ -95,8 +101,7 @@ impl RoutingOutcome {
pub fn answered(selected_model_id: ModelId, mut request: Request, response: Response) -> Self {
request.llm_request.model = Some(selected_model_id.to_string());
Self {
- selected_model_id,
- fallback_models: Vec::new(),
+ selected_model_ids: vec![selected_model_id],
request,
response: Some(response),
}
@@ -194,7 +199,7 @@ impl Driver {
let selected_model = result
.as_ref()
.ok()
- .map(|outcome| outcome.selected_model_id.clone());
+ .and_then(|outcome| outcome.selected_model_id().ok().cloned());
let step = result.map(|outcome| Step::Done(Box::new(outcome)));
self.step_tx
.send(step)
@@ -472,16 +477,15 @@ mod tests {
request(),
);
- assert_eq!(outcome.selected_model_id, "selected");
assert_eq!(
- outcome.fallback_models,
- target_set(&["fallback-one", "fallback-two"])
+ outcome.selected_model_ids,
+ target_set(&["selected", "fallback-one", "fallback-two"])
);
assert_eq!(outcome.request.model_id().as_deref(), Some("selected"));
assert!(outcome.response.is_none());
let outcome = RoutingOutcome::route_to("only".into(), Vec::new(), request());
- assert!(outcome.fallback_models.is_empty());
+ assert_eq!(outcome.selected_model_ids, target_set(&["only"]));
let outcome = RoutingOutcome::answered(
"answered".into(),
@@ -492,9 +496,8 @@ mod tests {
},
);
- assert_eq!(outcome.selected_model_id, "answered");
+ assert_eq!(outcome.selected_model_ids, target_set(&["answered"]));
assert_eq!(outcome.request.model_id().as_deref(), Some("answered"));
- assert!(outcome.fallback_models.is_empty());
assert_eq!(
outcome
.response
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index 3fdb8c38b..aeb570988 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -53,7 +53,7 @@ pub(crate) async fn test_drive(
fulfill(Arc::clone(&routing_serve), call)
})
.await?;
- let selected_model = outcome.selected_model_id.clone();
+ let selected_model = outcome.selected_model_id()?.clone();
let response = match outcome.response {
Some(response) => response,
None => serve
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index e2d6a4eaf..0571a3e12 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -517,24 +517,17 @@ impl PyModelCall {
/// The terminal routing selection, rewritten request, and optional existing response.
#[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)]
struct PyRoutingOutcome {
- selected_model_id: String,
- fallback_models: Vec<String>,
+ selected_model_ids: Vec<String>,
request: Py<PyAny>,
response: Option<Py<PyAny>>,
}
#[pymethods]
impl PyRoutingOutcome {
- /// The model selected by the algorithm and tried first by the host.
+ /// Models selected by the algorithm, ordered best model first.
#[getter]
- fn selected_model_id(&self) -> &str {
- &self.selected_model_id
- }
-
- /// Additional models the host may try in order after an eligible failure.
- #[getter]
- fn fallback_models(&self) -> Vec<String> {
- self.fallback_models.clone()
+ fn selected_model_ids(&self) -> Vec<String> {
+ self.selected_model_ids.clone()
}
/// The normalized request after routing-time rewrites.
@@ -679,8 +672,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
}),
RustStep::Done(outcome) => {
let RoutingOutcome {
- selected_model_id,
- fallback_models,
+ selected_model_ids,
request,
response,
} = *outcome;
@@ -689,8 +681,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
outcome: Py::new(
py,
PyRoutingOutcome {
- selected_model_id: selected_model_id.to_string(),
- fallback_models: fallback_models
+ selected_model_ids: selected_model_ids
.iter()
.map(ToString::to_string)
.collect(),
diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs
index bcd175344..36ecb6410 100644
--- a/crates/switchyard-runner/src/runner.rs
+++ b/crates/switchyard-runner/src/runner.rs
@@ -99,13 +99,10 @@ impl Runner {
) -> Option<DecisionDescription> {
let route = self.route(model.as_str())?;
let resolve = |selected: &ModelId| route.decision_target(selected);
+ let mut model_ids = outcome.selected_model_ids.iter();
Some(DecisionDescription {
- selected: resolve(&outcome.selected_model_id)?,
- fallbacks: outcome
- .fallback_models
- .iter()
- .map(resolve)
- .collect::<Option<Vec<_>>>()?,
+ selected: resolve(model_ids.next()?)?,
+ fallbacks: model_ids.map(resolve).collect::<Option<Vec<_>>>()?,
})
}
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index eea1c7a2e..844931831 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -723,7 +723,7 @@ async fn decision(
match encode_aggregated_response(
&aggregate,
input_format,
- Some(outcome.selected_model_id.as_str()),
+ outcome.selected_model_id().ok().map(ModelId::as_str),
) {
Ok(response) => Some(response),
Err(error) => return server_error(error.to_string()),
diff --git a/examples/libsy.py b/examples/libsy.py
index 471ab5b55..fd6b8b684 100644
--- a/examples/libsy.py
+++ b/examples/libsy.py
@@ -68,10 +68,10 @@ async def main() -> None:
case Step.CallModel(call):
call.respond(await client.call(call.request, call.models[0]))
case Step.Done(outcome):
- print("Decision:", outcome.selected_model_id)
+ print("Decision:", outcome.selected_model_ids[0])
response = outcome.response or await client.call(
outcome.request,
- outcome.selected_model_id,
+ outcome.selected_model_ids[0],
)
match response:
case LlmResponse.Agg(aggregate_response):
diff --git a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py
index 1532204a1..333d9a80e 100644
--- a/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py
+++ b/examples/litellm/src/switchyard_litellm/plugins/switchyard_routing_plugin.py
@@ -181,7 +181,7 @@ class SwitchyardRoutingPlugin(LiteLLMRequestRewriter):
"Switchyard algorithm produced a response while routing, "
"which a LiteLLM routing plugin cannot return"
)
- selected = outcome.selected_model_id
+ selected = outcome.selected_model_ids[0]
if selected not in candidates:
raise ValueError(
f"Switchyard selected {selected!r}, which is not in LiteLLM's "
@@ -197,7 +197,7 @@ class SwitchyardRoutingPlugin(LiteLLMRequestRewriter):
]
context.signals["switchyard"] = {
"selected_model_id": selected,
- "fallback_models": outcome.fallback_models,
+ "fallback_models": outcome.selected_model_ids[1:],
}
if request_patch["set"] or request_patch.get("remove"):
context.signals["switchyard"]["request_patch"] = request_patch
diff --git a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py b/examples/litellm/tests/unit/test_switchyard_routing_plugin.py
index 3038f0bb5..e0cca081c 100644
--- a/examples/litellm/tests/unit/test_switchyard_routing_plugin.py
+++ b/examples/litellm/tests/unit/test_switchyard_routing_plugin.py
@@ -166,15 +166,14 @@ async def test_litellm_conversion_preserves_stage_tool_signal_input() -> None:
# ToolSignals reads only normalized messages, so exact equality protects every signal input.
assert _request(litellm_messages)["messages"] == original_messages
assert direct_outcome is not None
- assert direct_outcome.selected_model_id == SOL
- assert direct_outcome.fallback_models == [TERRA]
+ assert direct_outcome.selected_model_ids == [SOL, TERRA]
context = routing_context(litellm_messages)
await stage_plugin().run(context)
assert context.signals["switchyard"] == {
- "selected_model_id": direct_outcome.selected_model_id,
- "fallback_models": direct_outcome.fallback_models,
+ "selected_model_id": direct_outcome.selected_model_ids[0],
+ "fallback_models": direct_outcome.selected_model_ids[1:],
}
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index cb15172cc..1589d60ff 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -114,10 +114,7 @@ if TYPE_CHECKING:
@final
class RoutingOutcome:
@property
- def selected_model_id(self) -> str: ...
-
- @property
- def fallback_models(self) -> list[str]: ...
+ def selected_model_ids(self) -> list[str]: ...
@property
def request(self) -> dict[str, object]: ...
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 07dd3cda7..02ac614e4 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -80,10 +80,10 @@ async def run_algorithm(
if outcome.response is not None:
match outcome.response:
case LlmResponse.Agg(response):
- return outcome.selected_model_id, response
+ return outcome.selected_model_ids[0], response
case LlmResponse.Stream(_):
raise AssertionError("test helper expected an aggregate response")
- candidates = [outcome.selected_model_id, *outcome.fallback_models]
+ candidates = outcome.selected_model_ids
for index, target in enumerate(candidates):
candidate_request = {**outcome.request, "model": target}
client = (clients or {})[target]
@@ -93,7 +93,7 @@ async def run_algorithm(
if index + 1 == len(candidates):
raise
else:
- return outcome.selected_model_id, response
+ return outcome.selected_model_ids[0], response
raise AssertionError("algorithm stream ended without an outcome")
@@ -111,8 +111,7 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()
assert variants == ["done"]
assert outcome is not None
- assert outcome.selected_model_id == "fast"
- assert outcome.fallback_models == []
+ assert outcome.selected_model_ids == ["fast"]
assert outcome.response is None
response = await client.call(outcome.request)
assert client.calls[0]["model"] == "fast"
@@ -156,7 +155,7 @@ async def test_routing_call_accepts_a_streamed_response() -> None:
outcome = done
assert outcome is not None
- assert outcome.selected_model_id == "model-b"
+ assert outcome.selected_model_ids == ["model-b", "model-a"]
async def test_classifier_config_accepts_a_prompt_override() -> None: