Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(llm-class): fall back to capable when efficient overflows in escalation mode (#302)
Lin Jiae64c800acf59
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 05cc4052d..b9c86b62e 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -26,7 +26,7 @@ use crate::core::classifier::{Classification, Classifier, Score};use crate::core::state::{State, StateValue};use crate::{LibsyError, Result};use switchyard_protocol::{- AggLlmResponse, Context, LlmResponse, Request, Response, RoutedLlmClient,+ AggLlmResponse, Context, LlmClientError, LlmResponse, Request, Response, RoutedLlmClient,};const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md");@@ -471,7 +471,11 @@ impl Classifier<State> for EscalationClassifier {// `Classifier::score` takes no `ctx`, so inner calls use Context::default() and their// spans carry algorithm="" rather than the algorithm name. Known gap shared with the// task classifier's judge consultation.- let efficient_response = driver+ //+ // If the efficient model exceeds its context window, fall through to capable: returning+ // `(decisive(capable), None)` tells FallThrough::execute to call+ // call_llm_with_overflow_fallback with the capable target instead of surfacing the error.+ let efficient_response = match driver.call_llm_target(Context::default(),&self.efficient,@@ -481,7 +485,15 @@ impl Classifier<State> for EscalationClassifier {reasoning: Some("escalation classifier: efficient tier".into()),}),)- .await?;+ .await+ {+ Ok(r) => r,+ Err(LibsyError::ClientCall {+ source: LlmClientError::ContextWindowExceeded { .. },+ ..+ }) => return Ok((decisive(&self.capable.semantic_name), None)),+ Err(e) => return Err(e),+ };let agg = efficient_response.llm_response.into_agg()@@ -1725,4 +1737,51 @@ mod tests {assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));Ok(())}++ #[tokio::test]+ async fn escalation_classifier_falls_back_to_capable_when_efficient_overflows() -> Result<()> {+ // When the efficient model exceeds its context window inside score(), the classifier+ // must return capable rather than propagating the error — otherwise the client sees+ // HTTP 400 instead of a response from the strong model.+ struct OverflowClient;+ #[async_trait]+ impl RoutedLlmClient for OverflowClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, LlmClientError> {+ Err(LlmClientError::ContextWindowExceeded {+ model: decision.selected_model().to_string(),+ message: "prompt is too long".to_string(),+ })+ }+ }++ let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(c),+ };+ let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {+ judge_target: target("judge", QueuedClient::new([])), // must not be called+ efficient_target: target("efficient", Arc::new(OverflowClient)),+ capable_target: target("capable", QueuedClient::new(["capable answer"])),+ contract: ClassifierContractConfig::default(),+ config: EscalationJudgeConfig {+ confirmations: 1,+ ..EscalationJudgeConfig::default()+ },+ max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,+ })?);++ let (trace, response) = router.run(Context::default(), classify_request()).await?;++ assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));+ assert_eq!(+ response.llm_response.as_agg().map(completion_text),+ Some("capable answer".to_string())+ );+ Ok(())+ }}