Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore(libsy): better defaults for llm classifier (#253)
Ayush Agarwal99e835dc4c08
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rsindex 3f2f6d424..f19d848b6 100644--- a/crates/libsy/examples/research_agent.rs+++ b/crates/libsy/examples/research_agent.rs@@ -109,11 +109,7 @@ async fn main() -> Result<()> {strong,TaskClassifierConfig {base_threshold: BASE_THRESHOLD,- min_confidence: 0.0,- capability_elevated_floor: None,- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()},)?);diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rsindex 52fc51009..2d33cdeb2 100644--- a/crates/libsy/examples/research_agent_core.rs+++ b/crates/libsy/examples/research_agent_core.rs@@ -120,11 +120,7 @@ async fn main() -> Result<()> {strong,TaskClassifierConfig {base_threshold: BASE_THRESHOLD,- min_confidence: 0.0,- capability_elevated_floor: None,- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()},)?);diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 8e39f0d0f..57726d2bc 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -14,6 +14,7 @@ use serde::Deserialize;use switchyard_protocol::{LlmRequest, Message, OutputParams, Role, SimpleDecision};use super::fall_through::{DefaultTarget, FallThrough};+use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;use super::util::affinity::AffinityRouter;use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy};use super::util::llm_judge::{self, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};@@ -120,8 +121,8 @@ impl Judge for CapabilityJudge {model: request.llm_request.model.clone(),messages,output: OutputParams {+ max_output_tokens: Some(self.config.max_output_tokens),response_format: self.config.response_schema.clone(),- ..OutputParams::default()},..LlmRequest::default()},@@ -189,7 +190,7 @@ impl JudgePolicy for TaskClassifierPolicy {}}-#[derive(Clone, Debug, Default, Deserialize)]+#[derive(Clone, Debug, Deserialize)]/// Thresholds that control capability classifier routing.pub struct TaskClassifierConfig {/// Lowest solve probability that routes a supported task to the efficient target.@@ -214,6 +215,27 @@ pub struct TaskClassifierConfig {/// task, and the last `n` turns after it.#[serde(default)]pub recent_turn_window: Option<usize>,+ /// Maximum completion tokens available to the classifier verdict.+ #[serde(default = "default_judge_max_output_tokens")]+ pub max_output_tokens: u64,+}++const fn default_judge_max_output_tokens() -> u64 {+ DEFAULT_JUDGE_MAX_OUTPUT_TOKENS+}++impl Default for TaskClassifierConfig {+ fn default() -> Self {+ Self {+ base_threshold: 0.0,+ min_confidence: 0.0,+ capability_elevated_floor: None,+ session_affinity: false,+ message_hash_fallback: false,+ recent_turn_window: None,+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,+ }+ }}impl TaskClassifierConfig {@@ -251,6 +273,11 @@ impl TaskClassifierConfig {});}}+ if self.max_output_tokens == 0 {+ return Err(LibsyError::AlgorithmError {+ message: "max_output_tokens must be at least 1".to_string(),+ });+ }if self.message_hash_fallback && !self.session_affinity {return Err(LibsyError::AlgorithmError {message: "message_hash_fallback requires session_affinity".to_string(),@@ -418,7 +445,8 @@ impl LlmTaskClassifier {config: TaskClassifierConfig,) -> Result<Self> {config.validate()?;- let judge_config = Self::load_judge_config()?;+ let mut judge_config = Self::load_judge_config()?;+ judge_config.max_output_tokens = config.max_output_tokens;let targets = LlmTargetSet::new(vec![efficient_target.clone(), capable_target.clone()]);let session_affinity = config.session_affinity;let message_hash_fallback = config.message_hash_fallback;@@ -479,12 +507,19 @@ impl LlmTaskClassifier {efficient_target: LlmTarget,capable_target: LlmTarget,config: EscalationJudgeConfig,+ max_output_tokens: u64,) -> Result<Self> {let capable_name = capable_target.semantic_name.clone();let efficient_name = efficient_target.semantic_name.clone();let confirmations = config.confirmations;let esc = Arc::new(EscalationClassifier {- judge: escalation::build_judge(judge_target, capable_name, efficient_name, config)?,+ judge: escalation::build_judge(+ judge_target,+ capable_name,+ efficient_name,+ config,+ max_output_tokens,+ )?,capable: capable_target.clone(),efficient: efficient_target.clone(),confirmations,@@ -585,11 +620,7 @@ mod tests {fn test_config(base_threshold: f64) -> TaskClassifierConfig {TaskClassifierConfig {base_threshold,- min_confidence: 0.0,- capability_elevated_floor: None,- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()}}@@ -626,12 +657,17 @@ mod tests {#[derive(Default)]struct PerRequestClient {calls: Mutex<Vec<String>>,+ judge_max_output_tokens: Mutex<Vec<Option<u64>>>,}impl PerRequestClient {fn calls(&self) -> Vec<String> {self.calls.lock().clone()}++ fn judge_max_output_tokens(&self) -> Vec<Option<u64>> {+ self.judge_max_output_tokens.lock().clone()+ }}#[async_trait]@@ -645,6 +681,9 @@ mod tests {let model = decision.selected_model().to_string();self.calls.lock().push(model.clone());let completion = if model == "judge" {+ self.judge_max_output_tokens+ .lock()+ .push(request.llm_request.output.max_output_tokens);r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#.to_string()} else {format!("answer from {model}")@@ -753,6 +792,29 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> {+ let client = Arc::new(PerRequestClient::default());+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let router = Arc::new(LlmTaskClassifier::new(+ target("judge"),+ target("efficient"),+ target("capable"),+ TaskClassifierConfig {+ max_output_tokens: 512,+ ..test_config(TEST_THRESHOLD)+ },+ )?);++ router.run(Context::default(), classify_request()).await?;++ assert_eq!(client.judge_max_output_tokens(), vec![Some(512)]);+ Ok(())+ }+#[tokio::test]async fn classifier_config_enables_session_affinity() -> Result<()> {let client = Arc::new(PerRequestClient::default());@@ -857,26 +919,22 @@ mod tests {TaskClassifierConfig {base_threshold: 0.5,min_confidence: 1.1,- capability_elevated_floor: None,- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()},TaskClassifierConfig {base_threshold: 0.5,- min_confidence: 0.0,capability_elevated_floor: Some(0.5),- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()},TaskClassifierConfig {base_threshold: 0.5,- min_confidence: 0.0,- capability_elevated_floor: None,- session_affinity: false,message_hash_fallback: true,- recent_turn_window: None,+ ..TaskClassifierConfig::default()+ },+ TaskClassifierConfig {+ base_threshold: 0.5,+ max_output_tokens: 0,+ ..TaskClassifierConfig::default()},] {assert!(@@ -1035,6 +1093,10 @@ mod tests {judge_request.llm_request.output.response_format,judge.config.response_schema);+ assert_eq!(+ judge_request.llm_request.output.max_output_tokens,+ Some(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)+ );Ok(())}@@ -1183,6 +1245,7 @@ mod tests {confirmations: 1,..EscalationJudgeConfig::default()},+ DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,)?))}diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rsindex 2e39ecde9..e6adfbbe1 100644--- a/crates/libsy/src/algorithms/util.rs+++ b/crates/libsy/src/algorithms/util.rs@@ -8,3 +8,6 @@ pub(crate) mod prompts;pub(crate) mod stage;pub mod subagent;pub(crate) mod tool_signals;++/// Default completion budget for internal classifier and escalation judge calls.+pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096;diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rsindex c960c0971..8a377462a 100644--- a/crates/libsy/src/algorithms/util/escalation.rs+++ b/crates/libsy/src/algorithms/util/escalation.rs@@ -36,21 +36,12 @@ const FIRST_USER_CHARS: usize = 2_000;/// Backstop on the assembled transcript; the per-message caps normally bind first.const MAX_REQUEST_CHARS: usize = 18_000;-/// Completion budget for one judge reply. A runaway guard, not a budget: the benchmarked-/// verdict runs ~40 tokens, and a cap costs nothing it does not generate.-///-/// Headroom this wide is for a judge deliberately opted back into thinking — hosts ask judge-/// targets to answer without thinking first, but a judge that thinks and is cut off before-/// answering has no verdict in `content` and fails open on every turn.-const JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096;-/// The tuning surface for the trajectory judge.///-/// Deliberately small: the three settings an audit of the Python router's twenty-odd knobs-/// found to actually change routing outcomes. Everything else is a fixed invariant (the-/// constants above). Defaults are the benchmarked configuration.+/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant+/// (the constants above).#[derive(Clone, Debug, Deserialize)]-#[serde(default)]+#[serde(default, deny_unknown_fields)]pub struct EscalationJudgeConfig {/// Consecutive escalate verdicts required before a turn moves to the capable tier, which/// is also the turn that latches the session. Any decline clears the streak.@@ -124,7 +115,7 @@ impl Judge for EscalationJudge {],output: OutputParams {response_format: self.rubric.response_schema.clone(),- max_output_tokens: Some(JUDGE_MAX_OUTPUT_TOKENS),+ max_output_tokens: Some(self.rubric.max_output_tokens),},..LlmRequest::default()},@@ -171,9 +162,16 @@ pub(crate) fn build_judge(capable: String,efficient: String,config: EscalationJudgeConfig,+ max_output_tokens: u64,) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {config.validate()?;- let rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;+ if max_output_tokens == 0 {+ return Err(LibsyError::AlgorithmError {+ message: "max_output_tokens must be at least 1".to_string(),+ });+ }+ let mut rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;+ rubric.max_output_tokens = max_output_tokens;Ok(JudgeClassifier::new(EscalationJudge { rubric, config },judge_target,@@ -388,12 +386,27 @@ mod tests {// Bounded output, so a reasoning judge cannot run away mid-verdict.assert_eq!(built.llm_request.output.max_output_tokens,- Some(JUDGE_MAX_OUTPUT_TOKENS)+ Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS));assert!(built.llm_request.output.response_format.is_some());Ok(())}+ #[test]+ fn judge_request_uses_the_configured_completion_cap() -> Result<()> {+ let mut rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;+ rubric.max_output_tokens = 512;+ let judge = EscalationJudge {+ rubric,+ config: EscalationJudgeConfig::default(),+ };++ let built = judge.build_request(&State::default(), &request_at_turn(None, 1));++ assert_eq!(built.llm_request.output.max_output_tokens, Some(512));+ Ok(())+ }+#[test]fn conversation_turn_counts_assistant_replies() {// The judge is handed the transcript with this turn's reply already appended, whichdiff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 4e6d3e75c..c5d81e4bf 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -20,13 +20,27 @@ use crate::core::state::State;use crate::{LibsyError, Result};use switchyard_protocol::{Context, Decision, Request, Response};-#[derive(Clone, Debug, Default)]+use super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;++#[derive(Clone, Debug)]/// Prompt and structured-output contract for one judge.pub struct JudgeConfig {/// Prepended instructions that define what the judge evaluates.pub system_prompt: String,/// Optional provider response format that constrains the judge verdict.pub response_schema: Option<Value>,+ /// Maximum completion tokens available to the judge verdict.+ pub max_output_tokens: u64,+}++impl Default for JudgeConfig {+ fn default() -> Self {+ Self {+ system_prompt: String::new(),+ response_schema: None,+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,+ }+ }}/// Builds and parses requests for one algorithm-specific LLM judge.@@ -171,6 +185,7 @@ pub(crate) fn load_judge_config(Ok(JudgeConfig {system_prompt: prompt_template.replace("{{RESPONSE_SCHEMA}}", &prompt_schema),response_schema: Some(response_schema),+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,})}diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rsindex 64271716b..f03fee898 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy/tests/observability.rs@@ -1045,11 +1045,7 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_libstrong,TaskClassifierConfig {base_threshold: 0.5,- min_confidence: 0.0,- capability_elevated_floor: None,- session_affinity: false,- message_hash_fallback: false,- recent_turn_window: None,+ ..TaskClassifierConfig::default()},)?);diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 3131dae01..2377d6daa 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -125,7 +125,8 @@ impl PyTaskClassifierConfig {capability_elevated_floor=None,session_affinity=false,message_hash_fallback=false,- recent_turn_window=None+ recent_turn_window=None,+ max_output_tokens=4096))]#[allow(clippy::too_many_arguments)]fn new(@@ -135,6 +136,7 @@ impl PyTaskClassifierConfig {session_affinity: bool,message_hash_fallback: bool,recent_turn_window: Option<usize>,+ max_output_tokens: u64,) -> Self {Self {inner: TaskClassifierConfig {@@ -144,6 +146,7 @@ impl PyTaskClassifierConfig {session_affinity,message_hash_fallback,recent_turn_window,+ max_output_tokens,},}}diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 054202698..07ddc8ea7 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -348,6 +348,7 @@ fn build_algorithm(weak,strong,judge_config.clone(),+ classifier_config.max_output_tokens,),None => LlmTaskClassifier::new(classifier, weak, strong, classifier_config.clone()),}@@ -549,12 +550,34 @@ target = "weak"Ok(())}+ #[test]+ fn classifier_judge_completion_caps_are_configurable() -> ServerResult<()> {+ let capability = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 512",+ );+ server_state_from_toml(&capability)?;++ let escalation = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",+ );+ server_state_from_toml(&escalation)?;+ Ok(())+ }+#[test]fn rejects_unknown_fields_and_algorithm_types() {let unknown_field =VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");assert!(error_message(&unknown_field).contains("unknown field"));+ let nested_completion_cap = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",+ );+ assert!(error_message(&nested_completion_cap).contains("unknown field"));+let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");assert!(error_message(&unknown_algorithm).contains("unknown variant"));}@@ -598,6 +621,13 @@ target = "weak"VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),"base_threshold must be between 0 and 1",),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",+ ),+ "max_output_tokens must be at least 1",+ ),(VALID_CONFIG.replace("base_threshold = 0.5",