Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): add ordered target alternatives (#1208)

Alex Fournierec15e52142e2
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 9e024221e..5d79f9282 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -173,6 +173,8 @@ struct TargetEligibilityConfig {
#[serde(default)]
require_explicit_requirements: bool,
#[serde(default)]
+ target_alternatives: BTreeMap<String, Vec<String>>,
+ #[serde(default)]
scope: RequirementScope,
}
@@ -188,9 +190,21 @@ struct TargetPromptPolicy {
routing_answer_target: Option<ModelId>,
}
-type LaneTargets<'a> = BTreeMap<&'a str, &'a TargetConfig>;
+// Algorithms keep their role names; calls use the selected target's name.
+struct ResolvedLlmTarget<'a> {
+ name: &'a str,
+ config: &'a TargetConfig,
+}
+
+type LaneTargets<'a> = BTreeMap<&'a str, ResolvedLlmTarget<'a>>;
+type TargetOverrides<'a> = BTreeMap<&'a str, &'a str>;
type LaneDecisions<'a> = BTreeMap<&'a str, ResolvedDecisionTarget<'a>>;
+// Replacements bind logical names once; mappings never chain.
+fn bound_target<'a>(name: &'a str, overrides: &TargetOverrides<'a>) -> &'a str {
+ overrides.get(name).copied().unwrap_or(name)
+}
+
fn callable_targets(algorithm: &AlgorithmSpec) -> BTreeSet<&str> {
let mut targets: BTreeSet<_> = algorithm.callable_target_names().into_iter().collect();
for (_, decision) in algorithm.classifier_targets() {
@@ -201,13 +215,15 @@ fn callable_targets(algorithm: &AlgorithmSpec) -> BTreeSet<&str> {
targets
}
-// Equivalent requirements reuse the same configured targets and algorithm state.
+type TargetBindings = Vec<(String, String)>;
+
+// Cache by resolved targets, so equivalent requirements reuse algorithm-owned state.
pub(crate) struct EligibleLanes {
deployment: Arc<DeploymentConfig>,
route_name: String,
clients: Arc<BTreeMap<String, Arc<TranslatingLlmClient>>>,
decision_clients: Arc<BTreeMap<String, Arc<dyn RoutedDecisionClient>>>,
- lanes: Mutex<BTreeMap<Vec<String>, Arc<ExecutionLane>>>,
+ lanes: Mutex<BTreeMap<TargetBindings, Arc<ExecutionLane>>>,
}
impl EligibleLanes {
@@ -219,20 +235,45 @@ impl EligibleLanes {
standard: &Arc<ExecutionLane>,
) -> RunnerResult<Arc<ExecutionLane>> {
let route = &self.deployment.routes[&self.route_name];
+ let config = route
+ .eligibility
+ .as_ref()
+ .ok_or_else(|| RunnerError::configuration("route has no eligibility configuration"))?;
let callable = callable_targets(&route.algorithm);
- let allowed: BTreeSet<_> = callable
- .iter()
- .copied()
- .filter(|name| {
+ let mut allowed = BTreeSet::new();
+ let mut overrides = TargetOverrides::new();
+ for name in &callable {
+ let candidates = std::iter::once(*name).chain(
+ config
+ .target_alternatives
+ .get(*name)
+ .into_iter()
+ .flatten()
+ .map(String::as_str),
+ );
+ if let Some(target) = candidates.into_iter().find(|name| {
self.deployment
.target_labels(name)
.is_some_and(|labels| requirements.is_subset(labels))
- })
- .collect();
- if allowed.len() == callable.len() {
+ }) {
+ allowed.insert(*name);
+ if target != *name {
+ overrides.insert(*name, target);
+ }
+ }
+ }
+ if overrides.is_empty() && allowed.len() == callable.len() {
return Ok(Arc::clone(standard));
}
- let key = allowed.iter().map(|name| (*name).to_owned()).collect();
+ let key = allowed
+ .iter()
+ .map(|name| {
+ (
+ (*name).to_owned(),
+ bound_target(name, &overrides).to_owned(),
+ )
+ })
+ .collect();
let mut lanes = self.lanes.lock();
if let Some(lane) = lanes.get(&key) {
return Ok(Arc::clone(lane));
@@ -245,10 +286,27 @@ impl EligibleLanes {
message: error.to_string(),
}
})?;
- let targets = self.deployment.resolve_lane_targets(&algorithm)?;
+ let targets = self
+ .deployment
+ .resolve_lane_targets(&algorithm, &overrides)?;
+ if matches!(algorithm, AlgorithmSpec::Random { .. }) && !overrides.is_empty() {
+ let mut models = BTreeSet::new();
+ for name in algorithm.routing_target_names() {
+ let target = targets[name].name;
+ if !models.insert(target) {
+ return Err(LlmClientError::InvalidRequest {
+ message: format!(
+ "random route alternatives map multiple pool entries to target {target}; use distinct targets"
+ ),
+ }
+ .into());
+ }
+ }
+ }
let decisions = self.deployment.resolve_decision_targets(
&self.route_name,
&algorithm,
+ &overrides,
&self.decision_clients,
)?;
// Construction is synchronous; no model I/O or await occurs under this lock.
@@ -464,8 +522,8 @@ impl DeploymentConfig {
}
}
for (route_name, config) in &self.routes {
- if config.eligibility.is_some() {
- self.validate_eligibility(route_name, &config.algorithm)?;
+ if let Some(eligibility) = &config.eligibility {
+ self.validate_eligibility(route_name, &config.algorithm, eligibility)?;
}
}
Ok(())
@@ -571,9 +629,14 @@ impl DeploymentConfig {
{
model_properties_probes.insert(config.id.clone(), probe);
}
- let standard_targets = self.resolve_lane_targets(&config.algorithm)?;
- let decisions =
- self.resolve_decision_targets(route_name, &config.algorithm, &decision_clients)?;
+ let no_overrides = TargetOverrides::new();
+ let standard_targets = self.resolve_lane_targets(&config.algorithm, &no_overrides)?;
+ let decisions = self.resolve_decision_targets(
+ route_name,
+ &config.algorithm,
+ &no_overrides,
+ &decision_clients,
+ )?;
let (standard, caller_auth) = self.build_lane(
route_name,
&config.algorithm,
@@ -583,8 +646,17 @@ impl DeploymentConfig {
Some((algorithm, models)),
)?;
// Judges are included so a judge answer maps to its provider model too.
- let route_targets = config
- .callable_target_names()
+ let mut target_names = callable_targets(&config.algorithm);
+ if let Some(eligibility) = &config.eligibility {
+ target_names.extend(
+ eligibility
+ .target_alternatives
+ .values()
+ .flatten()
+ .map(String::as_str),
+ );
+ }
+ let route_targets = target_names
.into_iter()
.filter_map(|name| self.decision_target(name))
.collect();
@@ -616,18 +688,49 @@ impl DeploymentConfig {
Ok(runner)
}
- // Eligibility cannot forward caller credentials or authorize prefill checkpoints.
+ // Alternatives preserve the router's existing roles and never chain.
fn validate_eligibility(
&self,
route_name: &str,
algorithm: &AlgorithmSpec,
+ config: &TargetEligibilityConfig,
) -> RunnerResult<()> {
if !algorithm.supports_target_restrictions() {
return Err(RunnerError::configuration(format!(
"route {route_name} cannot use eligibility with prefill_router"
)));
}
- for name in callable_targets(algorithm) {
+ let callable = callable_targets(algorithm);
+ for (source, alternatives) in &config.target_alternatives {
+ if !callable.contains(source.as_str()) {
+ return Err(RunnerError::configuration(format!(
+ "route {route_name} target_alternatives source {source} is not used by the algorithm"
+ )));
+ }
+ if alternatives.is_empty() {
+ return Err(RunnerError::configuration(
+ "target alternatives must not be empty",
+ ));
+ }
+ for target in alternatives {
+ let same_kind = (self.targets.contains_key(source)
+ && self.targets.contains_key(target))
+ || (self.decision_targets.contains_key(source)
+ && self.decision_targets.contains_key(target));
+ if !same_kind {
+ return Err(RunnerError::configuration(format!(
+ "route {route_name} alternative {target} must exist and match the target kind of {source}"
+ )));
+ }
+ }
+ }
+ for name in callable.into_iter().chain(
+ config
+ .target_alternatives
+ .values()
+ .flatten()
+ .map(String::as_str),
+ ) {
if let Some(target) = self.targets.get(name)
&& self
.llm_clients
@@ -645,16 +748,27 @@ impl DeploymentConfig {
fn resolve_lane_targets<'a>(
&'a self,
algorithm: &'a AlgorithmSpec,
+ overrides: &TargetOverrides<'_>,
) -> RunnerResult<LaneTargets<'a>> {
let decisions = algorithm.decision_target_names();
callable_targets(algorithm)
.into_iter()
.filter(|name| !decisions.contains(name))
.map(|name| {
- let target = self.targets.get(name).ok_or_else(|| {
- RunnerError::configuration(format!("route references unknown target {name}"))
- })?;
- Ok((name, target))
+ let bound = bound_target(name, overrides);
+ let (resolved_name, config) =
+ self.targets.get_key_value(bound).ok_or_else(|| {
+ RunnerError::configuration(format!(
+ "route references unknown target {bound}"
+ ))
+ })?;
+ Ok((
+ name,
+ ResolvedLlmTarget {
+ name: resolved_name.as_str(),
+ config,
+ },
+ ))
})
.collect()
}
@@ -663,20 +777,22 @@ impl DeploymentConfig {
&'a self,
route_name: &str,
algorithm: &'a AlgorithmSpec,
+ overrides: &TargetOverrides<'_>,
clients: &'a BTreeMap<String, Arc<dyn RoutedDecisionClient>>,
) -> RunnerResult<LaneDecisions<'a>> {
algorithm
.decision_target_names()
.into_iter()
.map(|name| {
- let target = self.decision_targets.get(name).ok_or_else(|| {
+ let bound = bound_target(name, overrides);
+ let target = self.decision_targets.get(bound).ok_or_else(|| {
RunnerError::configuration(format!(
- "route {route_name} requires unknown decision target {name}"
+ "route {route_name} requires unknown decision target {bound}"
))
})?;
let client = clients.get(&target.decision_client).ok_or_else(|| {
RunnerError::configuration(format!(
- "decision target {name} references unknown decision client {}",
+ "decision target {bound} references unknown decision client {}",
target.decision_client
))
})?;
@@ -704,8 +820,8 @@ impl DeploymentConfig {
built
} else {
let model_ids = targets
- .keys()
- .map(|name| ((*name).to_owned(), ModelId::from(*name)))
+ .iter()
+ .map(|(name, target)| ((*name).to_owned(), ModelId::from(target.name)))
.chain(
decisions
.iter()
@@ -717,8 +833,8 @@ impl DeploymentConfig {
.map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
let models = models.with_provider_models(
targets
- .iter()
- .map(|(name, target)| (ModelId::from(*name), target.id.clone()))
+ .values()
+ .map(|target| (ModelId::from(target.name), target.config.id.clone()))
.collect(),
);
(algorithm, models)
@@ -913,21 +1029,32 @@ impl DeploymentConfig {
let mut decisions_by_model = HashMap::new();
for name in route.callable_target_names() {
if let Some(target) = decisions.get(name) {
- decisions_by_model.insert(target.model.clone(), Arc::clone(target.client));
+ if let Some(previous) =
+ decisions_by_model.insert(target.model.clone(), Arc::clone(target.client))
+ && !Arc::ptr_eq(&previous, target.client)
+ {
+ return Err(RunnerError::configuration(format!(
+ "route {route_name} maps decision model {} to different clients",
+ target.model
+ )));
+ }
continue;
}
let target = targets.get(name).ok_or_else(|| {
RunnerError::configuration(format!("route references unknown target {name}"))
})?;
- let client = clients.get(&target.llm_client).ok_or_else(|| {
+ let client = clients.get(&target.config.llm_client).ok_or_else(|| {
RunnerError::configuration(format!("target {name} has no constructed llm client"))
})?;
- let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
- RunnerError::configuration(format!(
- "target {name} references unknown llm client {}",
- target.llm_client
- ))
- })?;
+ let client_config =
+ self.llm_clients
+ .get(&target.config.llm_client)
+ .ok_or_else(|| {
+ RunnerError::configuration(format!(
+ "target {name} references unknown llm client {}",
+ target.config.llm_client
+ ))
+ })?;
if client_config.forward_auth {
let target_auth = client_config.format.caller_auth_kind();
has_mixed_families |= caller_auth.is_some_and(|kind| kind != target_auth);
@@ -935,7 +1062,7 @@ impl DeploymentConfig {
forwarding_origins.insert(client_config.base_url.0.origin().ascii_serialization());
}
let client: Arc<dyn RoutedLlmClient> = client.clone();
- by_model.insert(ModelId::from(name), client);
+ by_model.insert(ModelId::from(target.name), client);
}
// Only forwarding clients count: an api_key_env client sends the server's own key.
// Forwarding clients must share one credential family unless they all use the same
@@ -954,7 +1081,7 @@ impl DeploymentConfig {
let completion_targets = route
.routing_target_names()
.into_iter()
- .map(ModelId::from)
+ .map(|name| ModelId::from(targets[name].name))
.collect::<Vec<_>>();
let router = ClientRouter::new_with_decision_clients(
by_model,
@@ -969,8 +1096,8 @@ impl DeploymentConfig {
.into_iter()
.filter_map(|name| {
let target = &targets[name];
- (!self.llm_clients[&target.llm_client].fallback_on_rate_limit)
- .then(|| ModelId::from(name))
+ (!self.llm_clients[&target.config.llm_client].fallback_on_rate_limit)
+ .then(|| ModelId::from(target.name))
})
.collect(),
);
@@ -989,8 +1116,8 @@ impl DeploymentConfig {
let target = targets.get(name).ok_or_else(|| {
RunnerError::configuration(format!("route references unknown target {name}"))
})?;
- if let Some(prompt) = &target.system_prompt {
- prompts.insert(ModelId::from(name), prompt.clone());
+ if let Some(prompt) = &target.config.system_prompt {
+ prompts.insert(ModelId::from(target.name), prompt.clone());
}
}
let mut policy = TargetPromptPolicy {
@@ -1000,15 +1127,23 @@ impl DeploymentConfig {
let Some((response_name, dependency_name)) = route.routing_response_and_dependency() else {
return Ok(policy);
};
- if !policy.prompts.contains_key(response_name) {
+ let response = &targets[response_name];
+ if !policy.prompts.contains_key(response.name) {
return Ok(policy);
}
- if response_name == dependency_name {
+ if std::iter::once(dependency_name)
+ .chain(route.judge_target_names())
+ .any(|name| {
+ targets
+ .get(name)
+ .is_some_and(|dependency| dependency.name == response.name)
+ })
+ {
return Err(RunnerError::configuration(format!(
"route {route_name} cannot apply system_prompt to target {response_name}: it is also the routing-only target"
)));
}
- policy.routing_answer_target = Some(ModelId::from(response_name));
+ policy.routing_answer_target = Some(ModelId::from(response.name));
Ok(policy)
}
@@ -1058,12 +1193,16 @@ impl DeploymentConfig {
.filter_map(|(index, name)| {
let config = targets.get(name)?;
let target = self.build_auxiliary_target(
- name,
- config,
+ config.name,
+ config.config,
clients,
AuxiliaryOperation::AnthropicCountTokens,
)?;
- Some((count_tokens_priority(name, &config.id), index, target))
+ Some((
+ count_tokens_priority(config.name, &config.config.id),
+ index,
+ target,
+ ))
})
.min_by_key(|(priority, index, _)| (*priority, *index))
.map(|(_, _, target)| target)
@@ -1076,10 +1215,10 @@ impl DeploymentConfig {
clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
) -> Option<AuxiliaryTarget> {
route.routing_target_names().into_iter().find_map(|name| {
- let target = *targets.get(name)?;
+ let target = targets.get(name)?;
self.build_auxiliary_target(
- name,
- target,
+ target.name,
+ target.config,
clients,
AuxiliaryOperation::ResponsesInputTokens,
)
@@ -1631,6 +1770,229 @@ evidence = {{}}
Ok(())
}
+ fn alternative_config(algorithm: &str) -> String {
+ let globals = VALID_CONFIG.split("[routes.noop]").next().expect("globals");
+ let mut config = globals.to_string();
+ for name in ["weak", "strong", "classifier"] {
+ config.push_str(&format!(
+ "\n[targets.approved_{name}]\nid = 'approved/{name}'\nllm_client = 'primary'\nlabels = ['private']\n"
+ ));
+ }
+ config.push_str(&format!(
+ "\n[routes.test]\nid = 'route'\n{algorithm}\n[routes.test.eligibility.target_alternatives]\n"
+ ));
+ let spec: AlgorithmSpec = toml::from_str(algorithm).expect("test algorithm");
+ for name in spec
+ .callable_target_names()
+ .into_iter()
+ .collect::<BTreeSet<_>>()
+ {
+ config.push_str(&format!("{name} = ['approved_{name}']\n"));
+ }
+ config
+ }
+
+ // Alternatives preserve fixed roles and selected-target metadata.
+ #[tokio::test]
+ async fn alternatives_preserve_router_roles() -> RunnerResult<()> {
+ for (algorithm, initial) in [
+ ("type = 'passthrough'\ntarget = 'weak'", "weak"),
+ (
+ "type = 'stage_router'\ncapable_target = 'strong'\nefficient_target = 'weak'\npicker = 'capable_first'\nconfidence_threshold = 0.0",
+ "strong",
+ ),
+ ] {
+ let runner = Runner::from_toml(&alternative_config(algorithm))?;
+ let route = runner.route("route").expect("route");
+ let standard = route.decide(Request::default()).await?;
+ assert_eq!(standard.selected_model_id()?.as_str(), initial);
+ let mut request = Request::default();
+ require_target_labels(&mut request, ["private"])?;
+ let outcome = route.decide(request).await?;
+ assert_eq!(outcome.selected_model_id()?, &format!("approved_{initial}"));
+ assert!(
+ outcome
+ .selected_model_ids
+ .iter()
+ .all(|model| model.starts_with("approved_"))
+ );
+ let description = runner
+ .describe_decision(&ModelId::from("route"), &outcome)
+ .expect("mapped metadata");
+ assert_eq!(description.selected.target, format!("approved_{initial}"));
+ }
+ Ok(())
+ }
+
+ // Invalid references fail at load; unavailable approved bindings fail at admission.
+ #[tokio::test]
+ async fn alternatives_reject_invalid_bindings() -> RunnerResult<()> {
+ let config = alternative_config(
+ "type = 'auto'\ncapable_target = 'strong'\nefficient_target = 'weak'",
+ );
+ for (invalid, expected) in [
+ (
+ format!("{config}typo = ['approved_weak']\n"),
+ "not used by the algorithm",
+ ),
+ (
+ config.replace("weak = ['approved_weak']", "weak = ['missing']"),
+ "must exist and match the target kind",
+ ),
+ (
+ config.replace("weak = ['approved_weak']", "weak = []"),
+ "target alternatives must not be empty",
+ ),
+ (
+ config.replace(
+ "[llm_clients.primary]",
+ "[llm_clients.primary]\nforward_auth = true",
+ ),
+ "forwarded caller authentication",
+ ),
+ ] {
+ let message = error_message(&invalid);
+ assert!(
+ message.contains(expected),
+ "expected {expected:?}, got {message}"
+ );
+ }
+ let random =
+ alternative_config("type = 'random'\ntargets = ['weak', 'strong']\nweights = [0, 1]")
+ .replace("strong = ['approved_strong']", "strong = ['approved_weak']");
+ for (source, expected) in [
+ (
+ config.replace("weak = ['approved_weak']", "weak = ['strong']"),
+ "required target weak",
+ ),
+ (random, "multiple pool entries to target approved_weak"),
+ ] {
+ let runner = Runner::from_toml(&source)?;
+ let mut request = Request::default();
+ require_target_labels(&mut request, ["private"])?;
+ let error = runner
+ .route("route")
+ .expect("configured route")
+ .decide(request)
+ .await
+ .err()
+ .expect("invalid eligible bindings");
+ assert!(error.to_string().contains(expected), "{error}");
+ }
+ Runner::from_toml(&config.replace("id = 'approved/weak'", "id = 'weak/model'"))?;
+ let shared_model =
+ alternative_config("type = 'random'\ntargets = ['weak', 'strong']\nweights = [0, 1]")
+ .replace("id = 'approved/strong'", "id = 'approved/weak'");
+ let runner = Runner::from_toml(&shared_model)?;
+ let mut request = Request::default();
+ require_target_labels(&mut request, ["private"])?;
+ assert_eq!(
+ runner
+ .route("route")
+ .expect("route")
+ .decide(request)
+ .await?
+ .selected_model_id()?,
+ "approved_strong"
+ );
+ let wrong_kind = format!(
+ "{}\n[routes.typed.eligibility.target_alternatives]\nstandard = ['weak']\n",
+ decision_eligibility_config()
+ );
+ assert!(error_message(&wrong_kind).contains("must exist and match the target kind"));
+ Ok(())
+ }
+
+ // A secondary verifier must not inherit the answer target's prompt after binding.
+ #[tokio::test]
+ async fn alternatives_reject_prompted_vgr_secondary_judge_alias() -> RunnerResult<()> {
+ let source = alternative_config(
+ "type = 'vgr'\nefficient_target = 'weak'\ncapable_target = 'strong'\njudge_target = 'strong'\ncapable_judge_target = 'classifier'",
+ )
+ .replace(
+ "classifier = ['approved_classifier']",
+ "classifier = ['approved_weak']",
+ )
+ .replace(
+ "[targets.approved_weak]",
+ "[targets.approved_weak]\nsystem_prompt = 'Answer the user.'",
+ );
+ let runner = Runner::from_toml(&source)?;
+ let mut request = Request::default();
+ require_target_labels(&mut request, ["private"])?;
+ let error = runner
+ .route("route")
+ .expect("VGR route")
+ .decide(request)
+ .await
+ .err()
+ .expect("secondary judge alias must fail before model I/O");
+ assert!(matches!(error, RunnerError::Configuration { .. }));
+ assert!(error.to_string().contains("routing-only target"), "{error}");
+ Ok(())
+ }
+
+ // Decision IDs remain provider keys, so bindings cannot give one ID two clients.
+ #[tokio::test]
+ async fn alternatives_reject_ambiguous_composite_decision_clients() -> RunnerResult<()> {
+ let source = decision_eligibility_config().replace(
+ "decision_client = \"privacy\"\nlabels = [\"private\"]",
+ "decision_client = \"privacy\"",
+ );
+ let source = format!(
+ r#"{source}
+[decision_clients.other]
+format = "system_one"
+endpoint = "http://127.0.0.1:1/decisions"
+api_key_env = "PATH"
+timeout_ms = 1
+
+[decision_targets.secondary]
+id = "decision/secondary"
+decision_client = "privacy"
+
+[decision_targets.approved_primary]
+id = "decision/shared"
+decision_client = "privacy"
+labels = ["private"]
+
+[decision_targets.approved_secondary]
+id = "decision/shared"
+decision_client = "other"
+labels = ["private"]
+
+[routes.ambiguous]
+id = "switchyard/ambiguous"
+type = "composite"
+classifier = {{ target = "standard", classify_trigger = "user_turn", decision = {{ cutoff = 0.4, candidates = {{ a = "strong", b = "weak" }}, evidence = {{}} }} }}
+stage = {{ capable_target = "strong", efficient_target = "weak", confidence_threshold = 0.5 }}
+efficient = {{ classifier = {{ target = "secondary", decision = {{ cutoff = 0.4, candidates = {{ a = "strong", b = "weak" }}, evidence = {{}} }} }} }}
+
+[routes.ambiguous.eligibility.target_alternatives]
+standard = ["approved_primary"]
+secondary = ["approved_secondary"]
+"#
+ );
+ let runner = Runner::from_toml(&source)?;
+ let mut request = Request::default();
+ require_target_labels(&mut request, ["private"])?;
+ let error = runner
+ .route("switchyard/ambiguous")
+ .expect("composite route")
+ .decide(request)
+ .await
+ .err()
+ .expect("ambiguous decision clients must fail before model I/O");
+ assert!(matches!(error, RunnerError::Configuration { .. }));
+ assert!(
+ error
+ .to_string()
+ .contains("maps decision model decision/shared to different clients"),
+ "{error}"
+ );
+ Ok(())
+ }
+
#[test]
fn eligibility_rejects_unknown_fields_and_invalid_scope() {
let configured = eligible_pool_config();
@@ -1695,8 +2057,7 @@ evidence = {{}}
"cannot use eligibility with forwarded caller authentication",
),
(
- eligible_pool_config().replace(
- "type = \"random\"\ntargets = [\"weak\", \"strong\"]\nweights = [3.0, 1.0]\nseed = 7",
+ alternative_config(
"type = 'prefill_router'\ntargets = ['weak']\ncheckpoint = 'unused'",
),
"cannot use eligibility with prefill_router",
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
index f9e5ca2cb..125508f35 100644
--- a/crates/switchyard-runner/src/route.rs
+++ b/crates/switchyard-runner/src/route.rs
@@ -424,7 +424,6 @@ mod tests {
use crate::{Runner, require_target_labels};
const PASSTHROUGH: &str = "type = 'passthrough'\ntarget = 'fast'";
- const POOL: &str = "type = 'random'\ntargets = ['fast', 'approved_fast']";
fn config(algorithm: &str) -> String {
let mut source = String::from(
@@ -438,8 +437,13 @@ mod tests {
}
source.push_str(&format!(
"\n[routes.test]\nid = 'route'\n{algorithm}\n\
- [routes.test.eligibility]\nscope = 'request'\n"
+ [routes.test.eligibility]\nscope = 'request'\n\
+ [routes.test.eligibility.target_alternatives]\n"
));
+ let spec: crate::AlgorithmSpec = toml::from_str(algorithm).expect("test algorithm");
+ for target in spec.callable_target_names() {
+ source.push_str(&format!("{target} = ['approved_{target}']\n"));
+ }
source
}
@@ -452,7 +456,7 @@ mod tests {
#[test]
fn resolved_bindings_reuse_the_algorithm_and_standard_setup() {
- let source = config(POOL);
+ let source = config(PASSTHROUGH);
let runner = Runner::from_toml(&source).expect("eligible route");
let route = runner.route("route").expect("route");
let cloned = route.clone();
@@ -490,30 +494,35 @@ mod tests {
let source = format!(
"{}\n[routes.test.infra]\nmax_signal_age_ms = 1000\n\
min_waiting_requests = 4\nmax_capable_inflight = 2\n",
- config(algorithm).replace("labels = []", "labels = ['private']")
+ config(algorithm)
);
let runner = Runner::from_toml(&source).expect("eligible queue policy");
let route = runner.route("route").expect("route");
- let signals: InfraSignals = [("fast", 8), ("capable", 0)]
- .into_iter()
- .map(|(model, waiting)| {
- (
- model.into(),
- ModelInfraSignals {
- load: Some(PoolLoad {
- sampled_at: std::time::Instant::now(),
- min_waiting: waiting,
- max_waiting: waiting,
- running: 0,
- }),
- ..Default::default()
- },
- )
- })
- .collect();
+ let signals: InfraSignals = [
+ ("fast", 8),
+ ("capable", 0),
+ ("approved_fast", 8),
+ ("approved_capable", 0),
+ ]
+ .into_iter()
+ .map(|(model, waiting)| {
+ (
+ model.into(),
+ ModelInfraSignals {
+ load: Some(PoolLoad {
+ sampled_at: std::time::Instant::now(),
+ min_waiting: waiting,
+ max_waiting: waiting,
+ running: 0,
+ }),
+ ..Default::default()
+ },
+ )
+ })
+ .collect();
let mut stale =
- RoutingOutcome::route_to("outside".into(), vec![], request("request", &["private"]));
+ RoutingOutcome::route_to("fast".into(), vec![], request("request", &["private"]));
let error = route
.apply_infra(&mut stale, &signals)
.expect_err("stale decision");
@@ -522,9 +531,12 @@ mod tests {
"{error}"
);
- for labels in [&[] as &[&str], &["private"]] {
+ for (labels, initial, expected) in [
+ (&[] as &[&str], "fast", "capable"),
+ (&["private"], "approved_fast", "approved_capable"),
+ ] {
let mut outcome =
- RoutingOutcome::route_to("fast".into(), vec![], request("request", labels));
+ RoutingOutcome::route_to(initial.into(), vec![], request("request", labels));
assert!(
route
.apply_infra(&mut outcome, &signals)
@@ -532,7 +544,7 @@ mod tests {
);
assert_eq!(
outcome.selected_model_id().expect("selected model"),
- "capable"
+ expected
);
}
}
@@ -556,6 +568,10 @@ mod tests {
.expect("unmarked request");
assert_eq!(ordinary.selected_model_id().expect("model"), "fast");
require_target_labels(&mut request, ["private"]).expect("trusted requirements");
- assert!(route.decide(request).await.is_err());
+ let approved = route.decide(request).await.expect("approved request");
+ assert_eq!(
+ approved.selected_model_id().expect("model"),
+ "approved_fast"
+ );
}
}
diff --git a/crates/switchyard-runner/tests/target_eligibility.rs b/crates/switchyard-runner/tests/target_eligibility.rs
index 748f7cbd0..131ee10fb 100644
--- a/crates/switchyard-runner/tests/target_eligibility.rs
+++ b/crates/switchyard-runner/tests/target_eligibility.rs
@@ -5,7 +5,7 @@
use serde_json::{Value, json};
use switchyard_llm_client::AuxiliaryOperation;
-use switchyard_protocol::{LlmClientError, Request, text_request};
+use switchyard_protocol::{LlmClientError, ModelId, Request, text_request};
use switchyard_runner::{Runner, RunnerError, require_target_labels};
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
@@ -17,11 +17,16 @@ capable_target = "both_capable"
picker = "efficient_first"
confidence_threshold = 0.6
"#;
+const ALTERNATIVES: &str = r#"
+target_alternatives = { fast = ["private_fast", "both_fast"], capable = ["private_capable", "both_capable"] }
+"#;
+
fn config(endpoint: &str, algorithm: &str, policy: &str) -> String {
let targets = [
("fast", "cloud/fast", &["public"] as &[&str]),
("capable", "cloud/capable", &[]),
("private_fast", "private/fast", &["private"]),
+ ("private_capable", "private/capable", &["private"]),
("both_fast", "approved/fast", &["private", "customer_data"]),
(
"both_capable",
@@ -141,6 +146,129 @@ async fn configured_stage_keeps_answer_fallbacks_approved() {
server.verify().await;
}
+#[tokio::test]
+async fn ordered_alternatives_rebind_clients_settings_prompts_and_fallbacks() {
+ let primary = MockServer::start().await;
+ let approved = MockServer::start().await;
+ let algorithm = STAGE
+ .replace("both_fast", "fast")
+ .replace("both_capable", "capable");
+ let mut source = config(&primary.uri(), &algorithm, ALTERNATIVES);
+ let cases = [
+ (
+ &primary,
+ "fast",
+ "cloud/fast",
+ "minimal",
+ "PUBLIC PROMPT",
+ 200,
+ ),
+ (
+ &approved,
+ "private_fast",
+ "private/fast",
+ "low",
+ "PRIVATE FAST PROMPT",
+ 503,
+ ),
+ (
+ &approved,
+ "private_capable",
+ "private/capable",
+ "high",
+ "PRIVATE CAPABLE PROMPT",
+ 200,
+ ),
+ (
+ &approved,
+ "both_fast",
+ "approved/fast",
+ "medium",
+ "APPROVED PROMPT",
+ 200,
+ ),
+ ];
+ for (server, target, model, effort, prompt, status) in cases {
+ let client = if target == "fast" {
+ "primary"
+ } else {
+ "approved"
+ };
+ source = source.replace(
+ &format!("[targets.{target}]\nid = {model:?}\nllm_client = \"primary\""),
+ &format!(
+ "[targets.{target}]\nid = {model:?}\nllm_client = {client:?}\nreasoning_effort = {effort:?}\nsystem_prompt = {prompt:?}"
+ ),
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(body_partial_json(json!({
+ "model": model,
+ "reasoning_effort": effort,
+ "messages": [
+ {"role": "system", "content": prompt},
+ {"role": "user", "content": "Summarize these synthetic notes."}
+ ]
+ })))
+ .respond_with(if status == 200 {
+ completion(model, "answer")
+ } else {
+ ResponseTemplate::new(status)
+ })
+ .expect(1)
+ .mount(server)
+ .await;
+ }
+ source.push_str(&format!(
+ "\n[llm_clients.approved]\nformat = 'openai_chat'\nbase_url = '{}/v1'\nmax_retries = 0\nfailure_cooldown_ms = 0\n",
+ approved.uri()
+ ));
+ let runner = Runner::from_toml(&source).expect("ordered alternatives");
+ let route = runner.route("chat").expect("route");
+ for (task, labels, selected, served) in [
+ ("normal", None, "fast", "fast"),
+ (
+ "private",
+ Some(&["private"] as &[&str]),
+ "private_fast",
+ "private_capable",
+ ),
+ (
+ "pair",
+ Some(&["private", "customer_data"] as &[&str]),
+ "both_fast",
+ "both_fast",
+ ),
+ ] {
+ let outcome = route
+ .execute(request(task, labels), None)
+ .await
+ .expect("resolved answer and fallback");
+ assert_eq!(outcome.selected_model, selected);
+ assert_eq!(
+ outcome.response.served_model().map(|model| model.as_str()),
+ Some(served)
+ );
+ }
+ let alternative = ModelId::from("private_fast");
+ assert_eq!(
+ route
+ .target(&alternative)
+ .expect("alternative metadata")
+ .reasoning_effort
+ .as_deref(),
+ Some("low")
+ );
+ assert_eq!(route.provider_model(&alternative), "private/fast");
+ assert_eq!(models_called(&primary).await, ["cloud/fast"]);
+ assert_eq!(
+ models_called(&approved).await,
+ ["private/fast", "private/capable", "approved/fast"]
+ );
+ primary.verify().await;
+ approved.verify().await;
+}
+
#[tokio::test]
async fn filtered_targets_sharing_a_model_keep_their_own_settings() {
let server = MockServer::start().await;
@@ -344,18 +472,19 @@ fail_open = false
}
#[tokio::test]
-async fn configured_judges_and_auxiliary_calls_require_the_same_labels() {
+async fn alternative_judges_and_auxiliary_calls_require_the_same_labels() {
let server = MockServer::start().await;
let algorithm = r#"
type = "llm_classifier"
-classifier_target = "approved_judge"
-strong_target = "both_capable"
-weak_target = "both_fast"
+classifier_target = "judge"
+strong_target = "capable"
+weak_target = "fast"
base_threshold = 0.5
fail_open = false
"#;
- let runner =
- Runner::from_toml(&config(&server.uri(), algorithm, "")).expect("approved judge route");
+ let policy = r#"target_alternatives = { fast = ["both_fast"], capable = ["both_capable"], judge = ["approved_judge"] }"#;
+ let runner = Runner::from_toml(&config(&server.uri(), algorithm, policy))
+ .expect("approved judge alternatives");
respond(&server, "approved/judge", completion("approved/judge", &json!({
"crux": "bounded task", "primary_rule": "SUP-1", "capability_boundary": "supported", "p_solve": 0.9
}).to_string()), 1).await;
@@ -380,6 +509,9 @@ fail_open = false
server.verify().await;
server.reset().await;
+ let stage = STAGE
+ .replace("both_fast", "fast")
+ .replace("both_capable", "capable");
for (format, endpoint, operation) in [
(
"openai_responses",
@@ -392,7 +524,7 @@ fail_open = false
AuxiliaryOperation::AnthropicCountTokens,
),
] {
- let source = config(&server.uri(), STAGE, "")
+ let source = config(&server.uri(), &stage, ALTERNATIVES)
.replace("format = \"openai_chat\"", &format!("format = {format:?}"));
let runner = Runner::from_toml(&source).expect("auxiliary route");
let route = runner.route("chat").expect("route");
@@ -422,3 +554,84 @@ fail_open = false
server.reset().await;
}
}
+
+#[tokio::test]
+async fn typed_decision_alternatives_rebind_the_judge_and_candidate_targets() {
+ let primary = MockServer::start().await;
+ let approved = MockServer::start().await;
+ let algorithm = r#"
+type = "llm_classifier"
+classifier_target = "standard"
+strong_target = "capable"
+weak_target = "fast"
+fail_open = false
+"#;
+ let policy = r#"target_alternatives = { fast = ["both_fast"], capable = ["both_capable"], standard = ["approved_decision"] }"#;
+ let source = format!(
+ r#"{}
+[routes.chat.decision]
+cutoff = 0.5
+candidates = {{ capable = "capable", efficient = "fast" }}
+evidence = {{}}
+[decision_clients.standard]
+format = "openai_decisions"
+endpoint = "{}/v1/decisions"
+api_key_env = "PATH"
+timeout_ms = 2000
+[decision_clients.approved]
+format = "openai_decisions"
+endpoint = "{}/v1/decisions"
+api_key_env = "PATH"
+timeout_ms = 2000
+[decision_targets.standard]
+id = "cloud/decision"
+decision_client = "standard"
+[decision_targets.approved_decision]
+id = "approved/decision"
+decision_client = "approved"
+labels = ["private", "customer_data"]
+"#,
+ config(&primary.uri(), algorithm, policy),
+ primary.uri(),
+ approved.uri()
+ );
+ let runner = Runner::from_toml(&source).expect("typed judge alternatives");
+ Mock::given(method("POST"))
+ .and(path("/v1/decisions"))
+ .and(body_partial_json(json!({"model": "approved/decision"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "model": "approved/decision",
+ "answers": [{
+ "type": "choice", "name": "route", "choice": "no_advantage",
+ "probabilities": [
+ {"value": "advantage", "probability": 0.0},
+ {"value": "no_advantage", "probability": 1.0}
+ ]
+ }],
+ "usage": {
+ "input_tokens": 1, "output_tokens": 1, "total_tokens": 2,
+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
+ "output_tokens_details": {"reasoning_tokens": 0}
+ }
+ })))
+ .expect(1)
+ .mount(&approved)
+ .await;
+ respond(
+ &primary,
+ "approved/fast",
+ completion("approved/fast", "approved answer"),
+ 1,
+ )
+ .await;
+ let route = runner.route("chat").expect("route");
+ let outcome = route
+ .execute(request("typed", Some(&["private", "customer_data"])), None)
+ .await
+ .expect("approved typed judge and candidate");
+ assert_eq!(outcome.selected_model, "both_fast");
+ assert_eq!(models_called(&primary).await, ["approved/fast"]);
+ assert_eq!(models_called(&approved).await, ["approved/decision"]);
+ primary.verify().await;
+ approved.verify().await;
+}
diff --git a/docs/internal/target-eligibility.md b/docs/internal/target-eligibility.md
index bab3a7897..07b13d4a8 100644
--- a/docs/internal/target-eligibility.md
+++ b/docs/internal/target-eligibility.md
@@ -10,8 +10,12 @@ A target must carry every required label. Extra labels are fine, and label names
have no built-in meaning. Existing provider connections are omitted below.
```toml
-[targets.public]
-id = "cloud-model"
+[targets.fast]
+id = "cloud-small-model"
+llm_client = "cloud"
+
+[targets.capable]
+id = "cloud-large-model"
llm_client = "cloud"
[targets.approved_fast]
@@ -26,12 +30,19 @@ labels = ["private", "customer_data"]
[routes.assistant]
id = "switchyard/assistant"
-type = "random"
-targets = ["public", "approved_fast", "approved_capable"]
+type = "stage_router"
+efficient_target = "fast"
+capable_target = "capable"
+picker = "efficient_first"
+confidence_threshold = 0.6
[routes.assistant.eligibility]
require_explicit_requirements = true
scope = "request"
+
+[routes.assistant.eligibility.target_alternatives]
+fast = ["approved_fast"]
+capable = ["approved_capable"]
```
- `targets.<name>.labels` declares approvals; default `[]`.
@@ -42,15 +53,23 @@ scope = "request"
needs a collection; an explicitly empty one counts.
- `scope` is `"request"`, the default and only supported value. Requirements
cover this request and its internal model calls.
+- `target_alternatives` maps configured target names to ordered alternatives;
+ default `{}`. Keys name targets, not router roles. Alternatives must preserve
+ target kind.
+
+Switchyard prefers the original target when it carries every required label.
+Otherwise, it uses the first listed alternative that qualifies. Resolution is
+one step: it does not follow another target's alternatives. If no eligible target
+can fill a required role, the request fails before any provider call.
-The example's unrestricted pool contains all three targets. Requiring `private`
-keeps the two approved targets. Requiring both labels keeps the same two;
-requiring an unsupported label fails before any provider call. There is no table
-of accepted combinations.
+In the example, empty requirements keep `fast` and `capable`. Requiring `private`
+selects `approved_fast` and `approved_capable`. Requiring both labels selects the
+same two; requiring an unsupported label fails closed. There is no table of
+accepted combinations. Model pools are filtered by the same label requirements.
-Fixed-role routers keep their configured roles. Stage requires its efficient and
-capable targets to qualify; a judge must qualify too. An unapproved required role
-stops the request. This configuration does not search for replacement targets.
+Stage's efficient and capable roles remain distinct, and any configured judge
+must also qualify. Alternatives resolve approvals before execution; they are not
+a runtime fallback list. The algorithm keeps its existing answer fallback rules.
## Rust API
@@ -76,14 +95,14 @@ requests, even when they carry the same task ID.
```mermaid
flowchart TD
H["Trusted host: request + required labels"] --> A["Runner checks requirements"]
- A --> P["Filter configured pools; check required roles"]
+ A --> P["Resolve eligible roles; filter configured pools"]
P -->|"no valid setup"| E["Error: no provider call"]
P --> R["Existing routing algorithm"]
R --> G["libsy checks allowed model IDs"]
G --> C["LLM client executes approved calls and fallbacks"]
```
-Runner owns label matching and reuses setups that keep the same configured
+Runner owns label matching and reuses setups that resolve to the same configured
targets. Each setup keeps its algorithm, clients, model groups and auxiliary
targets together. Required judges, typed decisions, answer fallbacks and auxiliary
calls must qualify. Provider codecs keep their existing translation role.