Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(composite): support independent stage profiles (#1176)

ryan-lempka65c02c445285
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index d08dec1b3..8e8053e6a 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -112,6 +112,7 @@ pub async fn run(
&algorithm_name,
&outcome.request,
outcome.fallback_request.as_ref(),
+ &outcome.candidate_requests,
&outcome.selected_model_ids,
&observe,
)
@@ -317,14 +318,17 @@ async fn call_first_available<'a>(
algorithm: &str,
request: &'a Request,
fallback_request: Option<&'a Request>,
+ candidate_requests: &'a HashMap<ModelId, Request>,
models: &[ModelId],
observe: &(dyn Fn(ModelCallObservation) + Send + Sync),
) -> Result<(&'a Request, Response)> {
for (index, target) in models.iter().enumerate() {
- let request = match fallback_request {
- Some(fallback) if index > 0 => fallback,
- _ => request,
- };
+ let request = candidate_requests
+ .get(target)
+ .unwrap_or(match fallback_request {
+ Some(fallback) if index > 0 => fallback,
+ _ => request,
+ });
let prepared = clients.prepare_completion_request(request.clone(), target);
match call_one(
clients,
diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs
index 56d7faa85..6ac007306 100644
--- a/crates/libsy/src/algorithms/composite.rs
+++ b/crates/libsy/src/algorithms/composite.rs
@@ -18,13 +18,13 @@ use super::llm_class::{
};
use super::stage::{StageRouterConfig, build_stage_route};
use super::util::affinity::{ClassifyTrigger, evict_if_full, has_new_user_turn, retention_key};
-use super::util::stage::{Tier, fall_open_tier, set_fall_open};
+use super::util::stage::{PickerMode, Tier, fall_open_tier, set_fall_open};
use crate::core::algorithm::{Algorithm, Driver, RoutingIdentity};
-use crate::core::classifier::Classifier;
+use crate::core::classifier::{Classification, Classifier};
use crate::core::processor::{Event, Processor};
use crate::core::state::State;
use crate::{LibsyError, Result};
-use switchyard_protocol::{Category, Request};
+use switchyard_protocol::{Category, Request, Response};
const COMPOSITE: &str = "composite";
@@ -145,7 +145,36 @@ impl CompositeRouter {
/// A stage router carrying its own judge is allowed, but that judge sits ahead
/// of the fall-open tier and so answers most of the turns this one set a tier for.
pub fn new(config: CompositeRouterConfig) -> Result<Self> {
- let (trigger, message_hash_fallback) = match &config.judge {
+ let setter = Self::tier_setter(config.judge, config.stage.mode.default_tier())?;
+ let route = build_stage_route(config.stage)?
+ .with_name(COMPOSITE)
+ .with_processor(Arc::new(setter));
+ Ok(Self { route })
+ }
+
+ /// Builds two independent stage profiles selected by the judge.
+ pub fn new_with_profiles(
+ judge: CompositeJudgeConfig,
+ mut profile_0: StageRouterConfig,
+ mut profile_1: StageRouterConfig,
+ ) -> Result<Self> {
+ for profile in [&mut profile_0, &mut profile_1] {
+ if profile.mode == PickerMode::CapableFirst {
+ profile.efficient_hold_turns.get_or_insert(-1);
+ }
+ }
+ let setter = Self::tier_setter(judge, Tier::Efficient)?;
+ let profiles =
+ StageProfiles([build_stage_route(profile_0)?, build_stage_route(profile_1)?]);
+ let route = FallThrough::new_with_state()
+ .with_name(COMPOSITE)
+ .with_processor(Arc::new(setter))
+ .with_classifier(Arc::new(profiles));
+ Ok(Self { route })
+ }
+
+ fn tier_setter(judge: CompositeJudgeConfig, default_tier: Tier) -> Result<TierSetter> {
+ let (trigger, message_hash_fallback) = match &judge {
CompositeJudgeConfig::Capability(judge) => {
(judge.classify_trigger, judge.message_hash_fallback)
}
@@ -158,9 +187,9 @@ impl CompositeRouter {
message: "composite: classify_trigger must be user_turn or new_session".to_string(),
});
}
- let judge: Arc<dyn Classifier> = match config.judge {
+ let judge: Arc<dyn Classifier> = match judge {
CompositeJudgeConfig::Capability(judge) => {
- // Composite owns the trigger and retention; standalone affinity never runs.
+ // Composite owns the trigger and retained verdict.
Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
config: TaskClassifierConfig {
classify_trigger: ClassifyTrigger::EveryRequest,
@@ -173,17 +202,54 @@ impl CompositeRouter {
LlmTaskClassifier::custom_classifier(&config, true)?
}
};
- let setter = TierSetter {
+ Ok(TierSetter {
judge,
trigger,
message_hash_fallback,
- default_tier: config.stage.mode.default_tier(),
+ default_tier,
tiers: Mutex::new(HashMap::new()),
+ })
+ }
+}
+
+struct StageProfiles([FallThrough; 2]);
+
+#[async_trait]
+impl Classifier for StageProfiles {
+ async fn score(
+ &self,
+ state: &mut State,
+ request: &mut Request,
+ driver: &Driver,
+ ) -> Result<(Classification, Option<Response>)> {
+ let (profile, prefix) = match fall_open_tier(state) {
+ Some(Tier::Capable) => (&self.0[1], "composite.profile_1."),
+ _ => (&self.0[0], "composite.profile_0."),
};
- let route = build_stage_route(config.stage)?
- .with_name(COMPOSITE)
- .with_processor(Arc::new(setter));
- Ok(Self { route })
+ // Each profile retains its own holds, including in caller-carried state.
+ let mut profile_state = State {
+ extra: state
+ .extra
+ .iter()
+ .filter_map(|(key, value)| {
+ key.strip_prefix(prefix)
+ .map(|key| (key.to_string(), value.clone()))
+ })
+ .collect(),
+ ..State::default()
+ };
+ let result = profile.route(&mut profile_state, driver, request).await;
+ state.extra.retain(|key, _| !key.starts_with(prefix));
+ state.extra.extend(
+ profile_state
+ .extra
+ .into_iter()
+ .map(|(key, value)| (format!("{prefix}{key}"), value)),
+ );
+ state.tool_signals = profile_state.tool_signals;
+ state.target_prompts = profile_state.target_prompts;
+ let (score, response) = result?;
+ Ok((Classification::Scores(vec![score]), response))
}
}
@@ -211,7 +277,6 @@ mod tests {
use switchyard_protocol::{Category, LlmClientError, Message, ModelId, Role};
use super::*;
- use crate::algorithms::util::stage::PickerMode;
use crate::algorithms::util::tier_fixtures::{JUDGE, Recorder, turn_request};
use crate::core::testing::test_drive_with_models;
@@ -348,6 +413,78 @@ mod tests {
})?))
}
+ #[tokio::test]
+ async fn profiles_keep_their_own_holds_and_prompts_when_the_judge_switches() -> Result<()> {
+ for (has_session_id, picker, first_target, first_prompt) in [
+ (false, PickerMode::EfficientFirst, "weak", "EF efficient"),
+ (true, PickerMode::EfficientFirst, "weak", "EF efficient"),
+ (true, PickerMode::CapableFirst, "strong", "EF capable"),
+ ] {
+ let mut efficient = StageRouterConfig::new(picker, 0.5);
+ efficient.capable_hold_turns = -1;
+ efficient.capable_system_prompt = Some("EF capable".into());
+ efficient.efficient_system_prompt = Some("EF efficient".into());
+ let mut capable = StageRouterConfig::new(PickerMode::CapableFirst, 0.0);
+ capable.capable_hold_turns = 0;
+ capable.capable_system_prompt = Some("CF capable".into());
+ capable.efficient_system_prompt = Some("CF efficient".into());
+ let router = Arc::new(CompositeRouter::new_with_profiles(
+ CompositeJudgeConfig::Capability(TaskClassifierConfig {
+ judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
+ base_threshold: 0.5,
+ ..Default::default()
+ }),
+ classify_trigger: ClassifyTrigger::UserTurn,
+ ..Default::default()
+ }),
+ efficient,
+ capable,
+ )?);
+ let recorder = Arc::new(Recorder::default());
+ let mut write = turn_request(false);
+ if let switchyard_protocol::ContentBlock::ToolCall(call) =
+ &mut write.llm_request.messages[1].content[0]
+ {
+ call.name = "Write".into();
+ }
+ let mut state = State::default();
+ for (p_solve, request, target, prompt) in [
+ (1.0, user_turn_request(), first_target, first_prompt),
+ (1.0, turn_request(true), "strong", "EF capable"),
+ (0.0, user_turn_request(), "strong", "CF capable"),
+ (0.0, write, "weak", "CF efficient"),
+ (1.0, user_turn_request(), "strong", "EF capable"),
+ (0.0, user_turn_request(), "weak", "CF efficient"),
+ ] {
+ *recorder.judge_p_solve.lock() = p_solve;
+ let request = if has_session_id {
+ request
+ } else {
+ unkeyed(request)
+ };
+ let outcome = crate::core::testing::test_drive_with_state(
+ router.clone(),
+ request,
+ runtime_models(),
+ state,
+ recorder.serve(),
+ )
+ .await?;
+ assert_eq!(outcome.selected_model_id()?.as_str(), target);
+ assert_eq!(outcome.request.llm_request.instructions.len(), 1);
+ assert_eq!(
+ outcome.request.llm_request.instructions[0].content,
+ vec![switchyard_protocol::ContentBlock::Text {
+ text: prompt.into()
+ }]
+ );
+ state = outcome.state;
+ }
+ assert_eq!(recorder.judge_calls(), 4);
+ }
+ Ok(())
+ }
+
#[test]
fn rejects_every_request_as_a_trigger() {
for judge in [
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 9a7286601..6b0a01906 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -133,19 +133,23 @@ impl FallThrough {
// it, later components see the rewrite, and the final value reaches the model.
let mut request = request;
let session_state = self.session_state(&request);
- let (score, served) = match session_state {
+ let (score, served, prompts) = match session_state {
Some(state) => {
let mut state = state.lock().await;
- self.route(&mut state, &driver, &mut request).await?
+ let (score, served) = self.route(&mut state, &driver, &mut request).await?;
+ (score, served, std::mem::take(&mut state.target_prompts))
}
None => {
let mut state = driver.take_state();
- let routed = self.route(&mut state, &driver, &mut request).await?;
+ let (score, served) = self.route(&mut state, &driver, &mut request).await?;
+ let prompts = std::mem::take(&mut state.target_prompts);
driver.set_state(state);
- routed
+ (score, served, prompts)
}
};
+ tracing::info!(algorithm=self.name, target=%score.target, confidence=score.confidence, "Model selected");
+
// A classifier that already called a model — because deciding required one, and that
// call also answers the turn — hands its response back here, so the turn is not paid
// for twice.
@@ -166,7 +170,21 @@ impl FallThrough {
fallback_models.push(candidate.clone());
}
}
- Ok(RoutingOutcome::route_to(target, fallback_models, request))
+ let mut outcome = RoutingOutcome::route_to(target, fallback_models, request);
+ if !prompts.is_empty() {
+ for target in outcome.selected_model_ids.iter().skip(1) {
+ let mut request = outcome.request.clone();
+ request.llm_request.model = Some(target.to_string());
+ if let Some(prompt) = prompts.get(target) {
+ super::util::prompts::prepend_system_prompt(&mut request, prompt);
+ }
+ outcome.candidate_requests.insert(target.clone(), request);
+ }
+ if let Some(prompt) = prompts.get(&outcome.selected_model_ids[0]) {
+ super::util::prompts::prepend_system_prompt(&mut outcome.request, prompt);
+ }
+ }
+ Ok(outcome)
}
}
}
@@ -192,7 +210,7 @@ impl FallThrough {
Some(Arc::clone(&session.state))
}
- async fn route(
+ pub(crate) async fn route(
&self,
state: &mut State,
driver: &Driver,
@@ -223,10 +241,9 @@ impl FallThrough {
});
};
- // 3. Resolve the target and log the choice.
+ // 3. Resolve the target.
algorithm::ensure_model_is_target(driver.models_for(&Category::Any), &score.target)?;
let target = score.target.clone();
- tracing::info!(algorithm=self.name, target=%score.target, confidence=score.confidence, "Model selected");
// 4. Post-decision replay: every processor sees the selection so stateful ones
// can bind it, and may rewrite the outbound request (e.g. add a target prompt).
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index df649dfad..77795593d 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -20,7 +20,6 @@ use async_trait::async_trait;
use super::fall_through::FallThrough;
use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig};
use super::util::affinity::ClassifyTrigger;
-use super::util::prompts::prepend_system_prompt;
use super::util::stage::{
DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, Tier, fall_open_tier,
record_routing_decision,
@@ -43,18 +42,26 @@ struct TierPromptProcessor {
#[async_trait]
impl Processor for TierPromptProcessor {
- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Decision {
- request, category, ..
+ driver,
+ category,
+ selected_model_id,
+ ..
} = event
{
- let prompt = match category {
- Some(Category::Capable) => self.capable.as_deref(),
- Some(Category::Efficient) => self.efficient.as_deref(),
- _ => None,
- };
- if let Some(prompt) = prompt {
- prepend_system_prompt(request, prompt);
+ state.target_prompts.clear();
+ for (tier, prompt) in [
+ (Category::Capable, &self.capable),
+ (Category::Efficient, &self.efficient),
+ ] {
+ if let Some(prompt) = prompt {
+ for target in driver.models_for(&tier) {
+ if target != selected_model_id || category.as_ref() == Some(&tier) {
+ state.target_prompts.insert(target.clone(), prompt.clone());
+ }
+ }
+ }
}
}
Ok(())
@@ -68,6 +75,7 @@ impl Processor for TierPromptProcessor {
struct SourceStamp {
inner: Arc<dyn Classifier>,
source: DecisionSource,
+ judge: Option<switchyard_protocol::ModelId>,
}
#[async_trait]
@@ -78,6 +86,11 @@ impl Classifier for SourceStamp {
request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
+ let scoped = self
+ .judge
+ .as_ref()
+ .map(|judge| driver.for_judge(judge.clone()));
+ let driver = scoped.as_ref().unwrap_or(driver);
let (classification, served) = self.inner.score(state, request, driver).await?;
// An abstaining classifier passes the turn on, so it is not its to claim.
if let Some(winner) = classification.argmax(false)? {
@@ -122,6 +135,8 @@ impl Classifier for FallOpen {
/// The capability judge a stage router falls through to.
pub struct LlmFallback {
+ /// Overrides the runtime judge for this stage configuration.
+ pub target: Option<switchyard_protocol::ModelId>,
/// Judge configuration. Its `recent_turn_window` counts trailing messages, while
/// this router's `recent_window` counts tool results.
/// Note: `classify_trigger` and `message_hash_fallback` have no effect here. The judge
@@ -198,17 +213,6 @@ impl StageRouter {
/// fallback judge configured to retain verdicts: it runs on every ambiguous
/// turn, so `classify_trigger` and `message_hash_fallback` would do nothing.
pub fn new(config: StageRouterConfig) -> Result<Self> {
- if let Some(fallback) = &config.llm_fallback
- && (fallback.config.classify_trigger != ClassifyTrigger::EveryRequest
- || fallback.config.message_hash_fallback)
- {
- return Err(LibsyError::AlgorithmError {
- message: "classify_trigger and message_hash_fallback have no effect on the \
- stage fallback judge, which runs on every ambiguous turn; remove \
- them, or use a composite route to judge once per user turn"
- .to_string(),
- });
- }
Ok(Self {
route: build_stage_route(config)?,
})
@@ -233,6 +237,17 @@ impl Algorithm for StageRouter {
/// Wires the cascade the wrapper drives. Exposed so a composition above can
/// stack a prelude onto it.
pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough> {
+ if let Some(fallback) = &config.llm_fallback
+ && (fallback.config.classify_trigger != ClassifyTrigger::EveryRequest
+ || fallback.config.message_hash_fallback)
+ {
+ return Err(LibsyError::AlgorithmError {
+ message: "classify_trigger and message_hash_fallback have no effect on the \
+ stage fallback judge, which runs on every ambiguous turn. Remove \
+ them, or use a composite route to judge once per user turn"
+ .to_string(),
+ });
+ }
for (name, threshold) in [
("confidence_threshold", config.confidence_threshold),
("capable_first_threshold", config.capable_first_threshold),
@@ -285,12 +300,14 @@ pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough
config: fallback.config,
})?),
source: DecisionSource::LlmClassifier,
+ judge: fallback.target,
}));
}
// Nothing behind this, so no turn is left unrouted.
router = router.with_classifier(Arc::new(SourceStamp {
inner: Arc::new(fall_open),
source: DecisionSource::FallOpen,
+ judge: None,
}));
if config.capable_system_prompt.is_some() || config.efficient_system_prompt.is_some() {
router = router.with_processor(Arc::new(TierPromptProcessor {
@@ -357,6 +374,7 @@ mod tests {
let stamp = SourceStamp {
inner,
source: DecisionSource::LlmClassifier,
+ judge: None,
};
let (driver, mut rx) = Driver::new("test", Arc::new(crate::RuntimeModels::default()));
stamp
@@ -429,6 +447,7 @@ mod tests {
fn rejects_an_out_of_range_judge_threshold() {
let mut config = config();
config.llm_fallback = Some(LlmFallback {
+ target: None,
config: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: -0.1,
@@ -452,6 +471,7 @@ mod tests {
] {
let mut config = config();
config.llm_fallback = Some(LlmFallback {
+ target: None,
config: TaskClassifierConfig {
classify_trigger: trigger,
message_hash_fallback,
@@ -699,6 +719,7 @@ mod tests {
*recorder.judge_p_solve.lock() = p_solve;
let mut c = config();
c.llm_fallback = Some(LlmFallback {
+ target: None,
config: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: 0.5,
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index ca90765c5..390bbef6e 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -263,6 +263,8 @@ pub struct RoutingOutcome {
/// rewrote `request` for the selected model only.
/// Content-policy denials do not advance to these fallback candidates.
pub fallback_request: Option<Request>,
+ /// Candidate-specific requests, including model-specific instructions.
+ pub candidate_requests: HashMap<ModelId, Request>,
/// A response produced while routing, or `None` when the client must make the answer call.
pub response: Option<Response>,
/// Outcome identity and optional algorithm evidence.
@@ -297,6 +299,7 @@ impl RoutingOutcome {
selected_model_ids,
request,
fallback_request: None,
+ candidate_requests: HashMap::new(),
response: None,
metadata: None,
state: State::default(),
@@ -311,6 +314,7 @@ impl RoutingOutcome {
selected_model_ids: vec![selected_model_id],
request,
fallback_request: None,
+ candidate_requests: HashMap::new(),
response: Some(response),
metadata: None,
state: State::default(),
@@ -363,6 +367,7 @@ pub struct Driver {
/// Which of those groups this driver reads.
scope: Scope,
+ judge: Option<ModelId>,
}
impl Driver {
@@ -385,6 +390,7 @@ impl Driver {
state: Arc::new(Mutex::new(State::default())),
models,
scope: Scope::Parent,
+ judge: None,
},
step_rx,
)
@@ -516,12 +522,24 @@ impl Driver {
/// The available models for this category, typically ordered best-first.
pub fn models_for(&self, category: &Category) -> &[ModelId] {
+ if *category == Category::Judge
+ && let Some(judge) = &self.judge
+ {
+ return std::slice::from_ref(judge);
+ }
match self.scope {
Scope::Parent => self.models.models_for(category),
Scope::Subagent => self.models.subagent_models_for(category),
}
}
+ pub(crate) fn for_judge(&self, judge: ModelId) -> Self {
+ Self {
+ judge: Some(judge),
+ ..self.clone()
+ }
+ }
+
/// The first available model for `category`.
pub fn first_model_for(&self, category: &Category) -> Result<&ModelId> {
self.models_for(category)
diff --git a/crates/libsy/src/core/state.rs b/crates/libsy/src/core/state.rs
index b21416d99..92068f896 100644
--- a/crates/libsy/src/core/state.rs
+++ b/crates/libsy/src/core/state.rs
@@ -3,7 +3,7 @@
//! Built-in state shared by processors and classifiers across a session.
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
use serde::{Deserialize, Serialize};
@@ -29,6 +29,9 @@ pub struct State {
/// so routers must treat absence as "no signal yet".
#[serde(skip)]
pub tool_signals: Option<ToolSignals>,
+ /// Per-model prompts for this routing outcome, cleared after routing.
+ #[serde(skip)]
+ pub target_prompts: HashMap<switchyard_protocol::ModelId, String>,
/// Algorithm-specific state keyed by stable internal names.
#[serde(flatten)]
pub extra: BTreeMap<String, StateValue>,
diff --git a/crates/switchyard-ffi/README.md b/crates/switchyard-ffi/README.md
index 9a20b330d..cf2f91436 100644
--- a/crates/switchyard-ffi/README.md
+++ b/crates/switchyard-ffi/README.md
@@ -29,14 +29,15 @@ Each accepted read delivers one step or an error through its callback:
- `call_model`: `{type, id, models, request}`
- `call_decision`: `{type, id, model, request}`
-- `done`: `{type, selected_model_ids, request, fallback_request, response, metadata}`
+- `done`: `{type, selected_model_ids, request, fallback_request, candidate_requests, response, metadata}`
A callback error or `done` ends the run. Outcomes retain ordered targets, the
prepared request, an optional buffered answer, and optional routing evidence.
The host executes the answer call when `response` is null.
Streamed routing answers are collected into a buffered response before `done`.
Cancellation interrupts collection; stream errors end the run with a callback error.
-When `fallback_request` is non-null, use it for candidates after the first.
+Use `candidate_requests[model_id]` when present. Otherwise, use `fallback_request`
+for candidates after the first when it is non-null.
Content-policy denials must not advance to those fallback candidates.
Wire formats are `openai_chat`, `openai_responses`, and `anthropic_messages`.
diff --git a/crates/switchyard-ffi/src/lib.rs b/crates/switchyard-ffi/src/lib.rs
index 270105906..72e238df4 100644
--- a/crates/switchyard-ffi/src/lib.rs
+++ b/crates/switchyard-ffi/src/lib.rs
@@ -127,6 +127,7 @@ enum RunStep<'a> {
selected_model_ids: &'a [ModelId],
request: &'a LlmRequest,
fallback_request: Option<&'a LlmRequest>,
+ candidate_requests: HashMap<&'a ModelId, &'a LlmRequest>,
response: Option<&'a AggLlmResponse>,
metadata: Option<OutcomeMetadataRef<'a>>,
},
@@ -303,6 +304,11 @@ impl RunState {
.fallback_request
.as_ref()
.map(|request| &request.llm_request),
+ candidate_requests: outcome
+ .candidate_requests
+ .iter()
+ .map(|(id, request)| (id, &request.llm_request))
+ .collect(),
response: response.as_ref(),
metadata: outcome
.metadata
diff --git a/crates/switchyard-ffi/src/tests.rs b/crates/switchyard-ffi/src/tests.rs
index 2cc53ceeb..f59e98740 100644
--- a/crates/switchyard-ffi/src/tests.rs
+++ b/crates/switchyard-ffi/src/tests.rs
@@ -250,6 +250,9 @@ fn completed_runs_preserve_fallback_requests() {
..Default::default()
},
);
+ outcome
+ .candidate_requests
+ .insert("efficient".into(), outcome.request.clone());
outcome.fallback_request = fallback.map(|llm_request| Request {
llm_request,
..Default::default()
@@ -265,6 +268,7 @@ fn completed_runs_preserve_fallback_requests() {
"rewritten"
);
assert_eq!(done.get("fallback_request"), Some(&expected));
+ assert_eq!(done["candidate_requests"]["efficient"], done["request"]);
}
}
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 5fc7f5230..f00620cc5 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -419,6 +419,7 @@ struct PyLlmFallback {
impl PyLlmFallback {
fn clone_core(&self, py: Python<'_>) -> PyResult<LlmFallback> {
Ok(LlmFallback {
+ target: None,
config: self.config.bind(py).try_borrow()?.clone_core(),
})
}
@@ -771,6 +772,8 @@ struct PyRoutingOutcome {
selected_model_ids: Vec<String>,
request: Py<PyAny>,
fallback_request: Option<Py<PyAny>>,
+ #[pyo3(get)]
+ candidate_requests: Py<PyAny>,
response: Option<Py<PyAny>>,
#[pyo3(get)]
metadata: Option<Py<PyOutcomeMetadata>>,
@@ -1038,6 +1041,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
selected_model_ids,
request,
fallback_request,
+ candidate_requests,
response,
metadata,
state: _,
@@ -1055,6 +1059,13 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
.map(ToString::to_string)
.collect(),
request: to_python(py, &request.llm_request)?,
+ candidate_requests: to_python(
+ py,
+ &candidate_requests
+ .into_iter()
+ .map(|(id, request)| (id.to_string(), request.llm_request))
+ .collect::<std::collections::HashMap<_, _>>(),
+ )?,
fallback_request: fallback_request
.map(|request| to_python(py, &request.llm_request))
.transpose()?,
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index c2ca4cc3a..10eebfd08 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -431,8 +431,12 @@ pub enum AlgorithmSpec {
},
/// Picks a tier per turn by scoring signals from recent tool results.
StageRouter {
+ capable_target: String,
+ efficient_target: String,
+ #[serde(default)]
+ capable_first_threshold: f64,
#[serde(flatten)]
- tiers: StageTierConfig,
+ settings: StageSettingsConfig,
/// Tier to use when the signals are not confident.
picker: PickerMode,
/// Judge consulted for turns the tool signals cannot decide.
@@ -458,6 +462,12 @@ pub enum AlgorithmSpec {
classifier: StageClassifierConfig,
/// The stage router the judge hands off to.
stage: StageTierConfig,
+ /// Settings for the profile selected by an efficient verdict.
+ #[serde(default)]
+ efficient: Option<Box<StageProfileConfig>>,
+ /// Settings for the profile selected by a capable verdict.
+ #[serde(default)]
+ capable: Option<Box<StageProfileConfig>>,
/// Separate policy for delegated sub-agent work.
#[serde(default)]
subagents: Option<SubagentRouteConfig>,
@@ -638,39 +648,75 @@ pub struct StageClassifierConfig {
pub max_output_tokens: u64,
}
-/// The tier pair and scoring settings shared by every stage-router-backed route.
-///
-/// `deny_unknown_fields` here is what rejects a typo on a flattened `stage_router`
-/// route, since the enum's own `deny_unknown_fields` does not apply through a flatten.
+/// The shared model pair and defaults for composite stage profiles.
#[derive(Clone, Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StageTierConfig {
- /// The capable tier.
pub capable_target: String,
- /// The efficient tier.
pub efficient_target: String,
- /// How much agreement a decisive pick needs, from 0 to 1.
- #[serde(default, alias = "efficient_first_threshold")]
- pub confidence_threshold: Option<f64>,
- /// Threshold after an outer classifier selects capable-first. Defaults to `0.0`.
#[serde(default)]
pub capable_first_threshold: f64,
- /// How many trailing tool results the signals are scored over.
- #[serde(default)]
+ #[serde(flatten)]
+ pub settings: StageSettingsConfig,
+}
+
+/// Settings shared by standalone stage routes and composite profiles.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct StageSettingsConfig {
+ #[serde(alias = "efficient_first_threshold")]
+ pub confidence_threshold: Option<f64>,
pub recent_turn_window: Option<usize>,
- /// Exact tool-name semantics added to the built-in stage vocabulary.
- #[serde(default)]
- pub tool_semantics: ToolSemantics,
- /// Requests to keep on the capable tier after an escalation.
- #[serde(default)]
+ pub tool_semantics: Option<ToolSemantics>,
pub capable_hold_turns: Option<i64>,
- /// Requests to keep on efficient after a switch. Defaults to `-1` for
- /// composite's capable-first policy, otherwise `0`. `-1` pins it.
- #[serde(default)]
pub efficient_hold_turns: Option<i64>,
- /// Notes handed to a tier when the router switches to it.
- #[serde(default)]
pub handoff_notes: Option<HandoffNoteConfig>,
+ pub capable_system_prompt: Option<String>,
+ pub efficient_system_prompt: Option<String>,
+}
+
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct StageProfileConfig {
+ pub classifier: Option<StageClassifierConfig>,
+ pub picker: Option<PickerMode>,
+ #[serde(flatten)]
+ pub settings: StageSettingsConfig,
+}
+
+impl StageSettingsConfig {
+ fn stage_config(&self, shared: &Self, mode: PickerMode, threshold: f64) -> StageRouterConfig {
+ let mut config =
+ StageRouterConfig::new(mode, self.confidence_threshold.unwrap_or(threshold));
+ config.recent_window = self.recent_turn_window.or(shared.recent_turn_window);
+ config.tool_semantics = self
+ .tool_semantics
+ .as_ref()
+ .or(shared.tool_semantics.as_ref())
+ .cloned()
+ .unwrap_or_default();
+ config.capable_hold_turns = self
+ .capable_hold_turns
+ .or(shared.capable_hold_turns)
+ .unwrap_or(2);
+ config.efficient_hold_turns = self.efficient_hold_turns.or(shared.efficient_hold_turns);
+ config.handoff_notes = self
+ .handoff_notes
+ .as_ref()
+ .or(shared.handoff_notes.as_ref())
+ .cloned();
+ config.capable_system_prompt = self
+ .capable_system_prompt
+ .as_ref()
+ .or(shared.capable_system_prompt.as_ref())
+ .cloned();
+ config.efficient_system_prompt = self
+ .efficient_system_prompt
+ .as_ref()
+ .or(shared.efficient_system_prompt.as_ref())
+ .cloned();
+ config
+ }
}
impl StageTierConfig {
@@ -684,7 +730,8 @@ impl StageClassifierConfig {
fn task_classifier_config(
&self,
route_name: &str,
- tiers: &StageTierConfig,
+ capable_target: &str,
+ efficient_target: &str,
targets: &BTreeMap<String, ModelId>,
) -> AlgorithmResult<TaskClassifierConfig> {
if self
@@ -699,8 +746,8 @@ impl StageClassifierConfig {
}
let config = LlmClassifierRouteConfig {
classifier_target: self.target.clone(),
- strong_target: Some(tiers.capable_target.clone()),
- weak_target: Some(tiers.efficient_target.clone()),
+ strong_target: Some(capable_target.to_string()),
+ weak_target: Some(efficient_target.to_string()),
base_threshold: self.base_threshold,
threshold_step: self.threshold_step,
fail_open: self.fail_open,
@@ -721,6 +768,19 @@ impl StageClassifierConfig {
}
}
+ fn stage_fallback(
+ &self,
+ route_name: &str,
+ capable: &str,
+ efficient: &str,
+ targets: &BTreeMap<String, ModelId>,
+ ) -> AlgorithmResult<LlmFallback> {
+ Ok(LlmFallback {
+ target: Some(resolve_target_model_id(route_name, &self.target, targets)?),
+ config: self.task_classifier_config(route_name, capable, efficient, targets)?,
+ })
+ }
+
fn custom_classifier_config(
&self,
route_name: &str,
@@ -761,23 +821,42 @@ impl StageClassifierConfig {
}
impl AlgorithmSpec {
- #[cfg(not(target_os = "wasi"))]
- pub(crate) fn decision_judge(&self) -> Option<(&str, &DecisionJudgeRouteConfig)> {
- match self {
- Self::LlmClassifier { config, .. } => config
- .decision
- .as_ref()
- .map(|decision| (config.classifier_target.as_str(), decision)),
- Self::Composite { classifier, .. }
- | Self::StageRouter {
- classifier: Some(classifier),
+ pub(crate) fn classifier_targets(
+ &self,
+ ) -> impl Iterator<Item = (&str, Option<&DecisionJudgeRouteConfig>)> {
+ let (primary, profiles) = match self {
+ Self::LlmClassifier { config, .. }
+ if !matches!(config.classifier_mode(), ClassifierMode::Custom) =>
+ {
+ (
+ Some((config.classifier_target.as_str(), config.decision.as_ref())),
+ [None, None],
+ )
+ }
+ Self::Composite {
+ classifier,
+ efficient,
+ capable,
..
- } => classifier
- .decision
- .as_ref()
- .map(|decision| (classifier.target.as_str(), decision)),
- _ => None,
- }
+ } => (
+ Some((classifier.target.as_str(), classifier.decision.as_ref())),
+ [efficient.as_deref(), capable.as_deref()],
+ ),
+ Self::StageRouter { classifier, .. } => (
+ classifier
+ .as_ref()
+ .map(|c| (c.target.as_str(), c.decision.as_ref())),
+ [None, None],
+ ),
+ _ => (None, [None, None]),
+ };
+ primary.into_iter().chain(
+ profiles
+ .into_iter()
+ .flatten()
+ .filter_map(|profile| profile.classifier.as_ref())
+ .map(|c| (c.target.as_str(), c.decision.as_ref())),
+ )
}
/// Completion targets in algorithm order; judge-only targets are excluded.
@@ -819,12 +898,12 @@ impl AlgorithmSpec {
.unwrap_or_default(),
},
Self::StageRouter {
- tiers, subagents, ..
+ capable_target,
+ efficient_target,
+ subagents,
+ ..
} => {
- let mut names = vec![
- tiers.capable_target.as_str(),
- tiers.efficient_target.as_str(),
- ];
+ let mut names = vec![capable_target.as_str(), efficient_target.as_str()];
if let Some(subagents) = subagents {
names.extend(subagents.routing_target_names());
}
@@ -893,12 +972,8 @@ impl AlgorithmSpec {
names.push(config.classifier_target.as_str());
}
}
- Self::StageRouter {
- classifier: Some(classifier),
- ..
- } => names.push(&classifier.target),
- Self::Composite { classifier, .. } => {
- names.push(&classifier.target);
+ Self::StageRouter { .. } | Self::Composite { .. } => {
+ names.extend(self.classifier_targets().map(|(target, _)| target));
}
Self::Advisor { advisor_target, .. } => names.push(advisor_target),
Self::Vgr { config } => {
@@ -952,14 +1027,17 @@ impl AlgorithmSpec {
classifier_runtime_model_names(config.validated_classifier_mode(route_name)?)
}
Self::StageRouter {
- tiers, classifier, ..
+ capable_target,
+ efficient_target,
+ classifier,
+ ..
} => {
let mut models = category_models([
- (Category::Capable, vec![tiers.capable_target.clone()]),
- (Category::Efficient, vec![tiers.efficient_target.clone()]),
+ (Category::Capable, vec![capable_target.clone()]),
+ (Category::Efficient, vec![efficient_target.clone()]),
(
Category::Any,
- vec![tiers.capable_target.clone(), tiers.efficient_target.clone()],
+ vec![capable_target.clone(), efficient_target.clone()],
),
]);
if let Some(classifier) = classifier {
@@ -978,10 +1056,14 @@ impl AlgorithmSpec {
vec![capable_target.clone(), efficient_target.clone()],
),
]),
- Self::Composite {
- classifier, stage, ..
- } => category_models([
- (Category::Judge, vec![classifier.target.clone()]),
+ Self::Composite { stage, .. } => category_models([
+ (
+ Category::Judge,
+ self.judge_target_names()
+ .into_iter()
+ .map(str::to_string)
+ .collect(),
+ ),
(Category::Capable, vec![stage.capable_target.clone()]),
(Category::Efficient, vec![stage.efficient_target.clone()]),
(
@@ -1631,49 +1713,34 @@ fn build_algorithm(
Ok(Arc::new(algorithm))
}
AlgorithmSpec::StageRouter {
- tiers,
+ capable_target,
+ efficient_target,
+ capable_first_threshold,
+ settings,
picker,
classifier,
subagents,
..
} => {
- let StageTierConfig {
- confidence_threshold,
- recent_turn_window,
- tool_semantics,
- capable_hold_turns,
- efficient_hold_turns,
- handoff_notes,
- ..
- } = tiers;
if matches!(picker, PickerMode::CapableFirst) {
tracing::warn!(
"stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."
);
}
- tiers.warn_identical_tiers(route_name);
- let threshold = confidence_threshold.ok_or_else(|| {
+ warn_identical_tiers(route_name, capable_target, efficient_target);
+ let threshold = settings.confidence_threshold.ok_or_else(|| {
AlgorithmConfigError::new(format!(
"stage_router route {route_name}: confidence_threshold is required"
))
})?;
- let mut config = StageRouterConfig::new(*picker, threshold);
- config.capable_first_threshold = tiers.capable_first_threshold;
- config.recent_window = *recent_turn_window;
- if let Some(turns) = capable_hold_turns {
- config.capable_hold_turns = *turns;
- }
- config.efficient_hold_turns = *efficient_hold_turns;
- config.tool_semantics = tool_semantics.clone();
- config.handoff_notes = handoff_notes.clone();
+ let mut config = settings.stage_config(settings, *picker, threshold);
+ config.capable_first_threshold = *capable_first_threshold;
// The judge is called through its own target, so it is not a routing
// destination and stays out of the tier pair.
config.llm_fallback = classifier
.as_ref()
.map(|classifier| {
- classifier
- .task_classifier_config(route_name, tiers, targets)
- .map(|config| LlmFallback { config })
+ classifier.stage_fallback(route_name, capable_target, efficient_target, targets)
})
.transpose()?;
let algorithm = StageRouter::new(config).map_err(|error| {
@@ -1702,32 +1769,67 @@ fn build_algorithm(
AlgorithmSpec::Composite {
classifier,
stage,
+ efficient,
+ capable,
subagents,
} => {
stage.warn_identical_tiers(route_name);
- let mut stage_config = StageRouterConfig::new(
- PickerMode::EfficientFirst,
- stage.confidence_threshold.unwrap_or(0.5),
- );
- stage_config.capable_first_threshold = stage.capable_first_threshold;
- stage_config.recent_window = stage.recent_turn_window;
- if let Some(turns) = stage.capable_hold_turns {
- stage_config.capable_hold_turns = turns;
- }
- stage_config.efficient_hold_turns = stage.efficient_hold_turns;
- stage_config.tool_semantics = stage.tool_semantics.clone();
- stage_config.handoff_notes = stage.handoff_notes.clone();
let judge = if matches!(classifier.mode, Some(ClassifierMode::Custom)) {
CompositeJudgeConfig::Custom(classifier.custom_classifier_config(route_name)?)
} else {
- CompositeJudgeConfig::Capability(
- classifier.task_classifier_config(route_name, stage, targets)?,
- )
+ CompositeJudgeConfig::Capability(classifier.task_classifier_config(
+ route_name,
+ &stage.capable_target,
+ &stage.efficient_target,
+ targets,
+ )?)
};
- let algorithm = CompositeRouter::new(CompositeRouterConfig {
- judge,
- stage: stage_config,
- })
+ let shared = &stage.settings;
+ let algorithm = if efficient.is_some() || capable.is_some() {
+ let defaults = StageProfileConfig::default();
+ let profiles = [
+ (
+ efficient,
+ PickerMode::EfficientFirst,
+ shared.confidence_threshold.unwrap_or(0.5),
+ ),
+ (
+ capable,
+ PickerMode::CapableFirst,
+ stage.capable_first_threshold,
+ ),
+ ]
+ .map(|(profile, picker, threshold)| -> AlgorithmResult<_> {
+ let profile = profile.as_deref().unwrap_or(&defaults);
+ let mut config = profile.settings.stage_config(
+ shared,
+ profile.picker.unwrap_or(picker),
+ threshold,
+ );
+ config.llm_fallback = profile
+ .classifier
+ .as_ref()
+ .map(|classifier| {
+ classifier.stage_fallback(
+ route_name,
+ &stage.capable_target,
+ &stage.efficient_target,
+ targets,
+ )
+ })
+ .transpose()?;
+ Ok(config)
+ });
+ let [profile_0, profile_1] = profiles;
+ CompositeRouter::new_with_profiles(judge, profile_0?, profile_1?)
+ } else {
+ let mut config = shared.stage_config(shared, PickerMode::EfficientFirst, 0.5);
+ config.capable_first_threshold = stage.capable_first_threshold;
+ CompositeRouter::new(CompositeRouterConfig {
+ judge,
+ stage: config,
+ })
+ }
.map_err(|error| {
AlgorithmConfigError::with_source(
format!("composite route {route_name}: {error}"),
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 285009d8f..5982d9417 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -330,9 +330,13 @@ impl DeploymentConfig {
) -> impl Iterator<Item = RunnerResult<(&str, &RouteConfig, AlgorithmAndModels)>> {
let targets = self.build_targets();
self.routes.iter().map(move |(route_name, config)| {
- let decision_judge = config.algorithm.decision_judge();
- for name in config.callable_target_names() {
- let (exists, kind) = if decision_judge.is_some_and(|(judge, _)| judge == name) {
+ let classifiers: Vec<_> = config.algorithm.classifier_targets().collect();
+ for (name, decision) in classifiers.iter().copied().chain(
+ config.callable_target_names().into_iter()
+ .filter(|name| !classifiers.iter().any(|(judge, _)| judge == name))
+ .map(|name| (name, None)),
+ ) {
+ let (exists, kind) = if decision.is_some() {
(self.decision_targets.contains_key(name), "decision")
} else {
(self.targets.contains_key(name), "LLM")
@@ -344,9 +348,10 @@ impl DeploymentConfig {
}
}
for name in config.routing_target_names().into_iter().chain(
- decision_judge
+ classifiers
.into_iter()
- .flat_map(|(_, judge)| judge.candidates.values().map(String::as_str)),
+ .filter_map(|(_, decision)| decision)
+ .flat_map(|judge| judge.candidates.values().map(String::as_str)),
) {
if !self.targets.contains_key(name) {
return Err(RunnerError::configuration(format!(
@@ -618,8 +623,8 @@ impl DeploymentConfig {
for name in route.callable_target_names() {
if route
.algorithm
- .decision_judge()
- .is_some_and(|(judge, _)| judge == name)
+ .classifier_targets()
+ .any(|(judge, decision)| judge == name && decision.is_some())
{
let target = &self.decision_targets[name];
decisions_by_model.insert(
@@ -1481,10 +1486,14 @@ new = ["send_message"]
#[test]
fn stage_and_composite_validate_hold_turns() {
+ let composite =
+ composite_config().replace("capable_hold_turns = 2", "capable_hold_turns = 0");
for source in [
stage_config(),
stage_config().replace("efficient_first", "capable_first"),
composite_config(),
+ format!("{composite}\n[routes.composed.efficient]\ncapable_hold_turns = 2"),
+ format!("{composite}\n[routes.composed.capable]\ncapable_hold_turns = 2"),
] {
for field in ["capable_hold_turns", "efficient_hold_turns"] {
for (turns, valid) in [
@@ -1531,7 +1540,7 @@ evidence = { description = "candidate comparison" }
] {
let deployment: DeploymentConfig = toml::from_str(&source).unwrap();
let spec = &deployment.routes["composed"].algorithm;
- assert_eq!(spec.decision_judge().unwrap().0, "tier_judge");
+ assert_eq!(spec.classifier_targets().next().unwrap().0, "tier_judge");
assert_eq!(
spec.build_with_runtime_models("composed", &deployment.build_targets())
.is_ok(),
diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs
index 23c21e1bc..955d54726 100644
--- a/crates/switchyard-runner/src/lib.rs
+++ b/crates/switchyard-runner/src/lib.rs
@@ -24,7 +24,8 @@ pub use config::{ConfiguredTarget, RouteTargets, TargetModel, api_key_variables,
pub use algorithm::{
AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, CategoryModelConfig, ClassifierMode,
ClassifierPolicyConfig, DecisionJudgeRouteConfig, LlmClassifierRouteConfig,
- StageClassifierConfig, SubagentRouteConfig, VgrModeConfig, VgrRouteConfig,
+ StageClassifierConfig, StageProfileConfig, StageSettingsConfig, SubagentRouteConfig,
+ VgrModeConfig, VgrRouteConfig,
};
#[cfg(not(target_os = "wasi"))]
pub use algorithm_factory::AlgorithmFactory;
diff --git a/crates/switchyard-runner/tests/composite_profiles.rs b/crates/switchyard-runner/tests/composite_profiles.rs
new file mode 100644
index 000000000..84d19a167
--- /dev/null
+++ b/crates/switchyard-runner/tests/composite_profiles.rs
@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde_json::json;
+use switchyard_protocol::{Request, text_request};
+use switchyard_runner::Runner;
+use wiremock::matchers::{body_partial_json, method, path};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+#[tokio::test]
+async fn profile_judges_and_fallback_prompts() -> Result<(), Box<dyn std::error::Error>> {
+ for (profile, inner_score) in [
+ ("efficient", None),
+ ("efficient", Some(0.0)),
+ ("capable", Some(1.0)),
+ ] {
+ let server = MockServer::start().await;
+ let mut source = format!(
+ r#"
+schema_version = 1
+[llm_clients.local]
+format = "openai_chat"
+base_url = "{}/v1"
+max_retries = 0
+[targets]
+judge = {{ id = "judge", llm_client = "local" }}
+strong = {{ id = "strong", llm_client = "local" }}
+weak = {{ id = "weak", llm_client = "local" }}
+inner = {{ id = "inner", llm_client = "local" }}
+[routes.auto]
+id = "auto"
+type = "composite"
+[routes.auto.classifier]
+target = "judge"
+base_threshold = 0.5
+classify_trigger = "user_turn"
+[routes.auto.stage]
+capable_target = "strong"
+efficient_target = "weak"
+[routes.auto.{profile}]
+capable_system_prompt = "CAPABLE PROMPT"
+efficient_system_prompt = "EFFICIENT PROMPT"
+"#,
+ server.uri()
+ );
+ if inner_score.is_some() {
+ source.push_str(&format!(
+ "\n[routes.auto.{profile}.classifier]\ntarget = 'inner'\nbase_threshold = 0.5\n"
+ ));
+ }
+ let runner = Runner::from_toml(&source)?;
+ let verdict = |score| {
+ format!(
+ r#"{{"crux":"task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":{score}}}"#
+ )
+ };
+ let outer = verdict(if profile == "efficient" { 1.0 } else { 0.0 });
+ let inner = verdict(inner_score.unwrap_or(0.0));
+ for (model, status, content) in [
+ ("judge", 200, outer.as_str()),
+ ("inner", 200, inner.as_str()),
+ ("weak", 503, "unavailable"),
+ ("strong", 200, "answer"),
+ ] {
+ Mock::given(method("POST")).and(path("/v1/chat/completions"))
+ .and(body_partial_json(json!({"model": model})))
+ .and(move |request: &wiremock::Request| {
+ let body: serde_json::Value = request.body_json().unwrap();
+ match model {
+ "weak" => body["messages"][0]["content"] == "EFFICIENT PROMPT",
+ "strong" => body["messages"][0]["content"] == "CAPABLE PROMPT",
+ _ => true,
+ }
+ })
+ .respond_with(ResponseTemplate::new(status).set_body_json(json!({
+ "id": "response", "object": "chat.completion", "model": model,
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}]
+ })))
+ .expect(match model {
+ "inner" => u64::from(inner_score.is_some()),
+ "weak" => u64::from(inner_score != Some(0.0)),
+ _ => 1,
+ }).mount(&server).await;
+ }
+ runner
+ .route("auto")
+ .unwrap()
+ .execute(
+ Request {
+ llm_request: text_request(None, "Help with this task"),
+ ..Default::default()
+ },
+ None,
+ )
+ .await?;
+ server.verify().await;
+ }
+ Ok(())
+}
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index cc8820264..6662046fc 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -426,6 +426,8 @@ the signals work and for tuning.
| `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. |
| `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. `efficient_first_threshold` is accepted as an alias. |
| `capable_first_threshold` | No | `0.0` | Threshold used after an outer classifier selects the capable tier. Only a [composite](#composite) route sets that tier, so this key has no effect on a standalone `stage_router` route. |
+| `capable_system_prompt` | No | unset | System prompt prepended for capable requests. |
+| `efficient_system_prompt` | No | unset | System prompt prepended for efficient requests. |
| `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. |
| `capable_hold_turns` | No | `2` | Requests kept on capable after a switch. Signal-driven capable decisions also start finite holds. Active holds skip signal processing and classification; `0` disables; `-1` pins after a switch. |
| `efficient_hold_turns` | No | `0` | Requests kept on efficient after a switch, even when errors appear. `0` disables; `-1` pins after a switch. |
@@ -545,6 +547,9 @@ confidence threshold. Tool signals still drive escalation and recovery. See
| `stage.handoff_notes` | No | unset | Optional guidance appended to forwarded requests. Uses the same fields and behavior as [stage-router handoff notes](../routing_algorithms/stage_router_routing.md#optional-handoff-notes). |
| `subagents` | No | unset | Nested policy used only for delegated sub-agent work. |
+Optional `efficient` and `capable` tables provide independent stage
+settings and prompts. See [independent stage profiles](../routing_algorithms/composite_routing.md#independent-stage-profiles).
+
For decision classifier configuration, see
[Decision model classifier](../routing_algorithms/composite_routing.md#decision-model-classifier).
The same decision settings are accepted by the optional `stage_router` classifier.
diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md
index 366f85801..c5949acbb 100644
--- a/docs/routing_algorithms/composite_routing.md
+++ b/docs/routing_algorithms/composite_routing.md
@@ -8,9 +8,9 @@ fit the same route type.
## What exists today
One pairing: a classifier selects the stage router's starting policy.
-An efficient classification selects efficient-first with `efficient_first_threshold`
+By default, an efficient classification selects efficient-first with `efficient_first_threshold`
(default `0.5`).
-A capable classification selects capable-first with `capable_first_threshold`
+A capable classification defaults to capable-first with `capable_first_threshold`
(default `0.0`). Both thresholds are configurable from `0.0` to `1.0`.
Tool signals can still change the tier
during execution. Both algorithms and the
@@ -64,6 +64,41 @@ nor a second `classifier`. The outer classifier supplies the default tier.
Custom `tool_semantics` remain stage-local and have the same additive behavior as
on a standalone stage route.
+### Independent stage profiles
+
+Add these tables beside `stage` to configure each profile over the shared target pair:
+
+```toml
+[routes.switchyard.efficient]
+picker = "efficient_first"
+confidence_threshold = 0.5
+efficient_hold_turns = 0
+capable_system_prompt = "Resolve the difficult step."
+efficient_system_prompt = "Carry out the next concrete step."
+
+[routes.switchyard.capable]
+picker = "capable_first"
+confidence_threshold = 0.2
+efficient_hold_turns = -1
+capable_system_prompt = "Plan the work and resolve uncertain steps."
+efficient_system_prompt = "Continue implementing the plan."
+```
+
+Both profiles use the [stage tuning and prompt settings](stage_router_routing.md#knob-reference).
+The judge's verdict selects `efficient` or `capable`. Their pickers default to
+efficient-first and capable-first respectively. Override `picker` to use either
+mode in either profile, including capable-first in both.
+Targets stay in `stage` and the profile selector stays in `classifier`.
+Each profile can also define its own `classifier` table, using the standalone
+stage classifier settings and target. It runs only when stage signals abstain.
+On model fallback, each candidate receives the active profile's prompt for its tier.
+Omitted settings inherit from `stage`, including the corresponding threshold.
+A supplied `tool_semantics` or `handoff_notes` table replaces the shared table.
+
+Either table enables two profiles with separate hold state. Switching back
+resumes that profile's holds. Inactive profiles do not consume hold turns.
+With neither table, existing behavior is preserved.
+
## Custom LLM classifier
To select a tier directly with your own prompt, keep the stage block and use:
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index c2a3f2b17..a17a51811 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -250,6 +250,8 @@ required columns.
| `picker` | required | `efficient_first` or `capable_first` (experimental). Tier used when the signals are not confident. |
| `confidence_threshold` | required | Corroboration a decisive pick needs, `0` to `1`. Alias `efficient_first_threshold`. |
| `capable_first_threshold` | `0.0` | Used only after an outer `composite` classifier selects the capable tier. No effect on a standalone route. |
+| `capable_system_prompt` | unset | System prompt prepended for capable requests. |
+| `efficient_system_prompt` | unset | System prompt prepended for efficient requests. |
| `recent_turn_window` | `3` | Trailing tool results the signals are computed over. |
| `capable_hold_turns` | `2` | Requests kept on capable after switching to it; signal-driven capable decisions also start finite holds. `0` disables; `-1` pins after a switch. |
| `efficient_hold_turns` | `0` | Requests kept on efficient after switching to it. `0` disables; `-1` pins after a switch. |
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index b6c6c9dde..da53ee214 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -178,6 +178,9 @@ if TYPE_CHECKING:
@property
def fallback_request(self) -> dict[str, object] | None: ...
+ @property
+ def candidate_requests(self) -> dict[str, dict[str, object]]: ...
+
@property
def response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 392db46ae..4bc0c5b10 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -94,6 +94,7 @@ async def run_algorithm(
await call.respond(serve(call))
case Step.Done(outcome):
assert isinstance(outcome.metadata, OutcomeMetadata)
+ assert isinstance(outcome.candidate_requests, dict)
assert UUID(outcome.metadata.outcome_id).version == 7
if outcome.response is not None:
match outcome.response:
@@ -103,7 +104,10 @@ async def run_algorithm(
raise AssertionError("test helper expected an aggregate response")
candidates = outcome.selected_model_ids
for index, target in enumerate(candidates):
- candidate_request = {**outcome.request, "model": target}
+ candidate_request = {
+ **outcome.candidate_requests.get(target, outcome.request),
+ "model": target,
+ }
client = (clients or {})[target]
try:
response = await client.call(candidate_request)
diff --git a/tests/test_stage_router_knob_docs.py b/tests/test_stage_router_knob_docs.py
index 4e995e3ef..d544e5516 100644
--- a/tests/test_stage_router_knob_docs.py
+++ b/tests/test_stage_router_knob_docs.py
@@ -82,7 +82,7 @@ def expected_keys() -> set[str]:
keys: set[str] = set()
for key in toml_keys(block(runner, "StageRouter {")):
keys.update(key.split("|"))
- for key in toml_keys(block(runner, "pub struct StageTierConfig {")):
+ for key in toml_keys(block(runner, "pub struct StageSettingsConfig {")):
field = key.split("|")[0]
if field == "tool_semantics":
keys.update(