Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): escalate on context overflow while buffering (#708)
nachiketb-nvidia86a99388561c
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rsindex 5d84e6113..94bc38e87 100644--- a/crates/libsy/src/algorithms/escalation.rs+++ b/crates/libsy/src/algorithms/escalation.rs@@ -107,10 +107,17 @@ impl Classifier<State> for EscalationClassifier {}Err(e) => return Err(e),};- // The call resolves when its stream handle arrives; transport can still fail while- // buffering. Fall back only for that availability failure and keep other errors typed.+ // The call resolves when its stream handle arrives; context and transport failures can+ // still occur while buffering.let agg = match efficient_response.llm_response.into_agg().await {Ok(agg) => agg,+ Err(LlmClientError::ContextWindowExceeded { .. }) => {+ driver.set_evidence(serde_json::json!({+ "source": "fallback",+ "reason_code": "context_window",+ }));+ return Ok((decisive(&capable), None));+ }Err(LlmClientError::Transport { .. }) => {driver.set_evidence(serde_json::json!({"source": "fallback",@@ -386,30 +393,38 @@ mod tests {#[tokio::test]async fn falls_back_to_capable_when_efficient_overflows() -> Result<()> {- let serve = |target: ModelId, _request: Request| async move {- match target.as_str() {- "efficient" => Err(LlmClientError::ContextWindowExceeded {- model: target,- message: "prompt is too long".to_string(),- }),- "judge" => panic!("the judge must not be consulted when efficient overflows"),- _ => Ok(reply("capable answer")),- }- };-- let (selected_model, response) = test_drive_with_models(- escalation_router()?,- classify_request(),- runtime_models(),- serve,- )- .await?;-- assert_eq!(selected_model, "capable");- assert_eq!(- response.llm_response.as_agg().map(completion_text),- Some("capable answer".to_string())- );+ for streamed in [false, true] {+ let serve = move |target: ModelId, _request: Request| async move {+ match target.as_str() {+ "efficient" if streamed => {+ Ok(streamed_then_error(LlmClientError::ContextWindowExceeded {+ model: target,+ message: "prompt is too long".to_string(),+ }))+ }+ "efficient" => Err(LlmClientError::ContextWindowExceeded {+ model: target,+ message: "prompt is too long".to_string(),+ }),+ "judge" => {+ panic!("the judge must not be consulted when efficient overflows")+ }+ _ => Ok(reply("capable answer")),+ }+ };+ let mut request = classify_request();+ request.llm_request.stream = streamed;++ let (selected_model, response) =+ test_drive_with_models(escalation_router()?, request, runtime_models(), serve)+ .await?;++ assert_eq!(selected_model, "capable");+ assert_eq!(+ response.llm_response.as_agg().map(completion_text),+ Some("capable answer".to_string())+ );+ }Ok(())}diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex 50c05665d..3263ff638 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -13,6 +13,7 @@ from switchyard.libsy import (Algorithm,ContextWindowExceededError,CustomClassifierConfig,+ EscalationClassifierConfig,LlmClassifierConfig,LlmResponse,OutcomeMetadata,@@ -463,6 +464,47 @@ async def test_context_window_failure_falls_back_to_the_next_model() -> None:]+async def test_escalation_falls_back_on_first_stream_context_overflow() -> None:+ async def overflow() -> AsyncIterator[dict[str, object]]:+ if False:+ yield {}+ raise ContextWindowExceededError("weak first-event context overflow")++ algorithm = algorithms.llm_classifier(+ LlmClassifierConfig.escalation(+ config=EscalationClassifierConfig(confirmations=1),+ )+ )+ calls: list[list[str]] = []+ outcome: RoutingOutcome | None = None+ models = {+ "judge": ["judge"],+ "efficient": ["weak"],+ "capable": ["strong"],+ "any": ["weak", "strong"],+ }++ async for step in algorithm.run_stream(+ {**request_body(), "stream": True}, models+ ):+ match step:+ case Step.CallModel(call):+ calls.append(call.models)+ call.respond(LlmResponse.Stream(overflow()))+ case Step.Done(done):+ outcome = done++ assert calls == [["weak"]]+ assert outcome is not None+ assert outcome.selected_model_ids == ["strong", "weak"]+ assert outcome.response is None+ assert outcome.metadata is not None+ assert outcome.metadata.evidence == {+ "source": "fallback",+ "reason_code": "context_window",+ }++async def test_stage_router_applies_additive_tool_semantics() -> None:"""Verify that configured mutation semantics reach stage-router scoring."""