Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): escalate on context overflow while buffering (#708)

nachiketb-nvidia86a99388561c
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs
index 5d84e6113..94bc38e87 100644
--- a/crates/libsy/src/algorithms/escalation.rs
+++ b/crates/libsy/src/algorithms/escalation.rs
@@ -107,10 +107,17 @@ impl Classifier<State> for EscalationClassifier {
}
Err(e) => return Err(e),
};
- // The call resolves when its stream handle arrives; transport can still fail while
- // buffering. Fall back only for that availability failure and keep other errors typed.
+ // The call resolves when its stream handle arrives; context and transport failures can
+ // still occur while buffering.
let agg = match efficient_response.llm_response.into_agg().await {
Ok(agg) => agg,
+ Err(LlmClientError::ContextWindowExceeded { .. }) => {
+ driver.set_evidence(serde_json::json!({
+ "source": "fallback",
+ "reason_code": "context_window",
+ }));
+ return Ok((decisive(&capable), None));
+ }
Err(LlmClientError::Transport { .. }) => {
driver.set_evidence(serde_json::json!({
"source": "fallback",
@@ -386,30 +393,38 @@ mod tests {
#[tokio::test]
async fn falls_back_to_capable_when_efficient_overflows() -> Result<()> {
- let serve = |target: ModelId, _request: Request| async move {
- match target.as_str() {
- "efficient" => Err(LlmClientError::ContextWindowExceeded {
- model: target,
- message: "prompt is too long".to_string(),
- }),
- "judge" => panic!("the judge must not be consulted when efficient overflows"),
- _ => Ok(reply("capable answer")),
- }
- };
-
- let (selected_model, response) = test_drive_with_models(
- escalation_router()?,
- classify_request(),
- runtime_models(),
- serve,
- )
- .await?;
-
- assert_eq!(selected_model, "capable");
- assert_eq!(
- response.llm_response.as_agg().map(completion_text),
- Some("capable answer".to_string())
- );
+ for streamed in [false, true] {
+ let serve = move |target: ModelId, _request: Request| async move {
+ match target.as_str() {
+ "efficient" if streamed => {
+ Ok(streamed_then_error(LlmClientError::ContextWindowExceeded {
+ model: target,
+ message: "prompt is too long".to_string(),
+ }))
+ }
+ "efficient" => Err(LlmClientError::ContextWindowExceeded {
+ model: target,
+ message: "prompt is too long".to_string(),
+ }),
+ "judge" => {
+ panic!("the judge must not be consulted when efficient overflows")
+ }
+ _ => Ok(reply("capable answer")),
+ }
+ };
+ let mut request = classify_request();
+ request.llm_request.stream = streamed;
+
+ let (selected_model, response) =
+ test_drive_with_models(escalation_router()?, request, runtime_models(), serve)
+ .await?;
+
+ assert_eq!(selected_model, "capable");
+ assert_eq!(
+ response.llm_response.as_agg().map(completion_text),
+ Some("capable answer".to_string())
+ );
+ }
Ok(())
}
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 50c05665d..3263ff638 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -13,6 +13,7 @@ from switchyard.libsy import (
Algorithm,
ContextWindowExceededError,
CustomClassifierConfig,
+ EscalationClassifierConfig,
LlmClassifierConfig,
LlmResponse,
OutcomeMetadata,
@@ -463,6 +464,47 @@ async def test_context_window_failure_falls_back_to_the_next_model() -> None:
]
+async def test_escalation_falls_back_on_first_stream_context_overflow() -> None:
+ async def overflow() -> AsyncIterator[dict[str, object]]:
+ if False:
+ yield {}
+ raise ContextWindowExceededError("weak first-event context overflow")
+
+ algorithm = algorithms.llm_classifier(
+ LlmClassifierConfig.escalation(
+ config=EscalationClassifierConfig(confirmations=1),
+ )
+ )
+ calls: list[list[str]] = []
+ outcome: RoutingOutcome | None = None
+ models = {
+ "judge": ["judge"],
+ "efficient": ["weak"],
+ "capable": ["strong"],
+ "any": ["weak", "strong"],
+ }
+
+ async for step in algorithm.run_stream(
+ {**request_body(), "stream": True}, models
+ ):
+ match step:
+ case Step.CallModel(call):
+ calls.append(call.models)
+ call.respond(LlmResponse.Stream(overflow()))
+ case Step.Done(done):
+ outcome = done
+
+ assert calls == [["weak"]]
+ assert outcome is not None
+ assert outcome.selected_model_ids == ["strong", "weak"]
+ assert outcome.response is None
+ assert outcome.metadata is not None
+ assert outcome.metadata.evidence == {
+ "source": "fallback",
+ "reason_code": "context_window",
+ }
+
+
async def test_stage_router_applies_additive_tool_semantics() -> None:
"""Verify that configured mutation semantics reach stage-router scoring."""