Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore(libsy): better defaults for llm classifier (#253)

Ayush Agarwal99e835dc4c08
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index 3f2f6d424..f19d848b6 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -109,11 +109,7 @@ async fn main() -> Result<()> {
strong,
TaskClassifierConfig {
base_threshold: BASE_THRESHOLD,
- min_confidence: 0.0,
- capability_elevated_floor: None,
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
},
)?);
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index 52fc51009..2d33cdeb2 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -120,11 +120,7 @@ async fn main() -> Result<()> {
strong,
TaskClassifierConfig {
base_threshold: BASE_THRESHOLD,
- min_confidence: 0.0,
- capability_elevated_floor: None,
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
},
)?);
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 8e39f0d0f..57726d2bc 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -14,6 +14,7 @@ use serde::Deserialize;
use switchyard_protocol::{LlmRequest, Message, OutputParams, Role, SimpleDecision};
use super::fall_through::{DefaultTarget, FallThrough};
+use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;
use super::util::affinity::AffinityRouter;
use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy};
use super::util::llm_judge::{self, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};
@@ -120,8 +121,8 @@ impl Judge for CapabilityJudge {
model: request.llm_request.model.clone(),
messages,
output: OutputParams {
+ max_output_tokens: Some(self.config.max_output_tokens),
response_format: self.config.response_schema.clone(),
- ..OutputParams::default()
},
..LlmRequest::default()
},
@@ -189,7 +190,7 @@ impl JudgePolicy for TaskClassifierPolicy {
}
}
-#[derive(Clone, Debug, Default, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
/// Thresholds that control capability classifier routing.
pub struct TaskClassifierConfig {
/// Lowest solve probability that routes a supported task to the efficient target.
@@ -214,6 +215,27 @@ pub struct TaskClassifierConfig {
/// task, and the last `n` turns after it.
#[serde(default)]
pub recent_turn_window: Option<usize>,
+ /// Maximum completion tokens available to the classifier verdict.
+ #[serde(default = "default_judge_max_output_tokens")]
+ pub max_output_tokens: u64,
+}
+
+const fn default_judge_max_output_tokens() -> u64 {
+ DEFAULT_JUDGE_MAX_OUTPUT_TOKENS
+}
+
+impl Default for TaskClassifierConfig {
+ fn default() -> Self {
+ Self {
+ base_threshold: 0.0,
+ min_confidence: 0.0,
+ capability_elevated_floor: None,
+ session_affinity: false,
+ message_hash_fallback: false,
+ recent_turn_window: None,
+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
+ }
+ }
}
impl TaskClassifierConfig {
@@ -251,6 +273,11 @@ impl TaskClassifierConfig {
});
}
}
+ if self.max_output_tokens == 0 {
+ return Err(LibsyError::AlgorithmError {
+ message: "max_output_tokens must be at least 1".to_string(),
+ });
+ }
if self.message_hash_fallback && !self.session_affinity {
return Err(LibsyError::AlgorithmError {
message: "message_hash_fallback requires session_affinity".to_string(),
@@ -418,7 +445,8 @@ impl LlmTaskClassifier {
config: TaskClassifierConfig,
) -> Result<Self> {
config.validate()?;
- let judge_config = Self::load_judge_config()?;
+ let mut judge_config = Self::load_judge_config()?;
+ judge_config.max_output_tokens = config.max_output_tokens;
let targets = LlmTargetSet::new(vec![efficient_target.clone(), capable_target.clone()]);
let session_affinity = config.session_affinity;
let message_hash_fallback = config.message_hash_fallback;
@@ -479,12 +507,19 @@ impl LlmTaskClassifier {
efficient_target: LlmTarget,
capable_target: LlmTarget,
config: EscalationJudgeConfig,
+ max_output_tokens: u64,
) -> Result<Self> {
let capable_name = capable_target.semantic_name.clone();
let efficient_name = efficient_target.semantic_name.clone();
let confirmations = config.confirmations;
let esc = Arc::new(EscalationClassifier {
- judge: escalation::build_judge(judge_target, capable_name, efficient_name, config)?,
+ judge: escalation::build_judge(
+ judge_target,
+ capable_name,
+ efficient_name,
+ config,
+ max_output_tokens,
+ )?,
capable: capable_target.clone(),
efficient: efficient_target.clone(),
confirmations,
@@ -585,11 +620,7 @@ mod tests {
fn test_config(base_threshold: f64) -> TaskClassifierConfig {
TaskClassifierConfig {
base_threshold,
- min_confidence: 0.0,
- capability_elevated_floor: None,
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
}
}
@@ -626,12 +657,17 @@ mod tests {
#[derive(Default)]
struct PerRequestClient {
calls: Mutex<Vec<String>>,
+ judge_max_output_tokens: Mutex<Vec<Option<u64>>>,
}
impl PerRequestClient {
fn calls(&self) -> Vec<String> {
self.calls.lock().clone()
}
+
+ fn judge_max_output_tokens(&self) -> Vec<Option<u64>> {
+ self.judge_max_output_tokens.lock().clone()
+ }
}
#[async_trait]
@@ -645,6 +681,9 @@ mod tests {
let model = decision.selected_model().to_string();
self.calls.lock().push(model.clone());
let completion = if model == "judge" {
+ self.judge_max_output_tokens
+ .lock()
+ .push(request.llm_request.output.max_output_tokens);
r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#.to_string()
} else {
format!("answer from {model}")
@@ -753,6 +792,29 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> {
+ let client = Arc::new(PerRequestClient::default());
+ let target = |name: &str| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: Some(client.clone()),
+ };
+ let router = Arc::new(LlmTaskClassifier::new(
+ target("judge"),
+ target("efficient"),
+ target("capable"),
+ TaskClassifierConfig {
+ max_output_tokens: 512,
+ ..test_config(TEST_THRESHOLD)
+ },
+ )?);
+
+ router.run(Context::default(), classify_request()).await?;
+
+ assert_eq!(client.judge_max_output_tokens(), vec![Some(512)]);
+ Ok(())
+ }
+
#[tokio::test]
async fn classifier_config_enables_session_affinity() -> Result<()> {
let client = Arc::new(PerRequestClient::default());
@@ -857,26 +919,22 @@ mod tests {
TaskClassifierConfig {
base_threshold: 0.5,
min_confidence: 1.1,
- capability_elevated_floor: None,
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
},
TaskClassifierConfig {
base_threshold: 0.5,
- min_confidence: 0.0,
capability_elevated_floor: Some(0.5),
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
},
TaskClassifierConfig {
base_threshold: 0.5,
- min_confidence: 0.0,
- capability_elevated_floor: None,
- session_affinity: false,
message_hash_fallback: true,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
+ },
+ TaskClassifierConfig {
+ base_threshold: 0.5,
+ max_output_tokens: 0,
+ ..TaskClassifierConfig::default()
},
] {
assert!(
@@ -1035,6 +1093,10 @@ mod tests {
judge_request.llm_request.output.response_format,
judge.config.response_schema
);
+ assert_eq!(
+ judge_request.llm_request.output.max_output_tokens,
+ Some(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
+ );
Ok(())
}
@@ -1183,6 +1245,7 @@ mod tests {
confirmations: 1,
..EscalationJudgeConfig::default()
},
+ DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
)?))
}
diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs
index 2e39ecde9..e6adfbbe1 100644
--- a/crates/libsy/src/algorithms/util.rs
+++ b/crates/libsy/src/algorithms/util.rs
@@ -8,3 +8,6 @@ pub(crate) mod prompts;
pub(crate) mod stage;
pub mod subagent;
pub(crate) mod tool_signals;
+
+/// Default completion budget for internal classifier and escalation judge calls.
+pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096;
diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs
index c960c0971..8a377462a 100644
--- a/crates/libsy/src/algorithms/util/escalation.rs
+++ b/crates/libsy/src/algorithms/util/escalation.rs
@@ -36,21 +36,12 @@ const FIRST_USER_CHARS: usize = 2_000;
/// Backstop on the assembled transcript; the per-message caps normally bind first.
const MAX_REQUEST_CHARS: usize = 18_000;
-/// Completion budget for one judge reply. A runaway guard, not a budget: the benchmarked
-/// verdict runs ~40 tokens, and a cap costs nothing it does not generate.
-///
-/// Headroom this wide is for a judge deliberately opted back into thinking — hosts ask judge
-/// targets to answer without thinking first, but a judge that thinks and is cut off before
-/// answering has no verdict in `content` and fails open on every turn.
-const JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096;
-
/// The tuning surface for the trajectory judge.
///
-/// Deliberately small: the three settings an audit of the Python router's twenty-odd knobs
-/// found to actually change routing outcomes. Everything else is a fixed invariant (the
-/// constants above). Defaults are the benchmarked configuration.
+/// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant
+/// (the constants above).
#[derive(Clone, Debug, Deserialize)]
-#[serde(default)]
+#[serde(default, deny_unknown_fields)]
pub struct EscalationJudgeConfig {
/// Consecutive escalate verdicts required before a turn moves to the capable tier, which
/// is also the turn that latches the session. Any decline clears the streak.
@@ -124,7 +115,7 @@ impl Judge for EscalationJudge {
],
output: OutputParams {
response_format: self.rubric.response_schema.clone(),
- max_output_tokens: Some(JUDGE_MAX_OUTPUT_TOKENS),
+ max_output_tokens: Some(self.rubric.max_output_tokens),
},
..LlmRequest::default()
},
@@ -171,9 +162,16 @@ pub(crate) fn build_judge(
capable: String,
efficient: String,
config: EscalationJudgeConfig,
+ max_output_tokens: u64,
) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
config.validate()?;
- let rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
+ if max_output_tokens == 0 {
+ return Err(LibsyError::AlgorithmError {
+ message: "max_output_tokens must be at least 1".to_string(),
+ });
+ }
+ let mut rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
+ rubric.max_output_tokens = max_output_tokens;
Ok(JudgeClassifier::new(
EscalationJudge { rubric, config },
judge_target,
@@ -388,12 +386,27 @@ mod tests {
// Bounded output, so a reasoning judge cannot run away mid-verdict.
assert_eq!(
built.llm_request.output.max_output_tokens,
- Some(JUDGE_MAX_OUTPUT_TOKENS)
+ Some(super::super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)
);
assert!(built.llm_request.output.response_format.is_some());
Ok(())
}
+ #[test]
+ fn judge_request_uses_the_configured_completion_cap() -> Result<()> {
+ let mut rubric = llm_judge::load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
+ rubric.max_output_tokens = 512;
+ let judge = EscalationJudge {
+ rubric,
+ config: EscalationJudgeConfig::default(),
+ };
+
+ let built = judge.build_request(&State::default(), &request_at_turn(None, 1));
+
+ assert_eq!(built.llm_request.output.max_output_tokens, Some(512));
+ Ok(())
+ }
+
#[test]
fn conversation_turn_counts_assistant_replies() {
// The judge is handed the transcript with this turn's reply already appended, which
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 4e6d3e75c..c5d81e4bf 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -20,13 +20,27 @@ use crate::core::state::State;
use crate::{LibsyError, Result};
use switchyard_protocol::{Context, Decision, Request, Response};
-#[derive(Clone, Debug, Default)]
+use super::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;
+
+#[derive(Clone, Debug)]
/// Prompt and structured-output contract for one judge.
pub struct JudgeConfig {
/// Prepended instructions that define what the judge evaluates.
pub system_prompt: String,
/// Optional provider response format that constrains the judge verdict.
pub response_schema: Option<Value>,
+ /// Maximum completion tokens available to the judge verdict.
+ pub max_output_tokens: u64,
+}
+
+impl Default for JudgeConfig {
+ fn default() -> Self {
+ Self {
+ system_prompt: String::new(),
+ response_schema: None,
+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
+ }
+ }
}
/// Builds and parses requests for one algorithm-specific LLM judge.
@@ -171,6 +185,7 @@ pub(crate) fn load_judge_config(
Ok(JudgeConfig {
system_prompt: prompt_template.replace("{{RESPONSE_SCHEMA}}", &prompt_schema),
response_schema: Some(response_schema),
+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
})
}
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 64271716b..f03fee898 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -1045,11 +1045,7 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
strong,
TaskClassifierConfig {
base_threshold: 0.5,
- min_confidence: 0.0,
- capability_elevated_floor: None,
- session_affinity: false,
- message_hash_fallback: false,
- recent_turn_window: None,
+ ..TaskClassifierConfig::default()
},
)?);
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 3131dae01..2377d6daa 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -125,7 +125,8 @@ impl PyTaskClassifierConfig {
capability_elevated_floor=None,
session_affinity=false,
message_hash_fallback=false,
- recent_turn_window=None
+ recent_turn_window=None,
+ max_output_tokens=4096
))]
#[allow(clippy::too_many_arguments)]
fn new(
@@ -135,6 +136,7 @@ impl PyTaskClassifierConfig {
session_affinity: bool,
message_hash_fallback: bool,
recent_turn_window: Option<usize>,
+ max_output_tokens: u64,
) -> Self {
Self {
inner: TaskClassifierConfig {
@@ -144,6 +146,7 @@ impl PyTaskClassifierConfig {
session_affinity,
message_hash_fallback,
recent_turn_window,
+ max_output_tokens,
},
}
}
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 054202698..07ddc8ea7 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -348,6 +348,7 @@ fn build_algorithm(
weak,
strong,
judge_config.clone(),
+ classifier_config.max_output_tokens,
),
None => LlmTaskClassifier::new(classifier, weak, strong, classifier_config.clone()),
}
@@ -549,12 +550,34 @@ target = "weak"
Ok(())
}
+ #[test]
+ fn classifier_judge_completion_caps_are_configurable() -> ServerResult<()> {
+ let capability = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 512",
+ );
+ server_state_from_toml(&capability)?;
+
+ let escalation = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",
+ );
+ server_state_from_toml(&escalation)?;
+ Ok(())
+ }
+
#[test]
fn rejects_unknown_fields_and_algorithm_types() {
let unknown_field =
VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");
assert!(error_message(&unknown_field).contains("unknown field"));
+ let nested_completion_cap = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",
+ );
+ assert!(error_message(&nested_completion_cap).contains("unknown field"));
+
let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
assert!(error_message(&unknown_algorithm).contains("unknown variant"));
}
@@ -598,6 +621,13 @@ target = "weak"
VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),
"base_threshold must be between 0 and 1",
),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",
+ ),
+ "max_output_tokens must be at least 1",
+ ),
(
VALID_CONFIG.replace(
"base_threshold = 0.5",