Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(llm-client): detect native sglang context-overflow messages (#426)
Francesco Parisio2d0d4b25a346
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex 4e1020ac0..fab446095 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -26,6 +26,8 @@ const OPENAI_OVERFLOW_PHRASES: &[&str] = &["context length is only","please reduce the length of the input","exceeds the maximum allowed input length",+ "exceeds the maximum allowed length",+ "is longer than the model's context length",];// Anthropic has no structured `error.code`, so detection is phrase-based only.@@ -450,6 +452,15 @@ mod tests {assert!(backend.is_context_overflow(r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#));+ // Native SGLang: top-level envelope (no `error` key), caught by the raw-body phrase match.+ // KV-pool rejection (managers/utils.py) and declared-context rejection+ // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.+ assert!(backend.is_context_overflow(+ r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#+ ));+ assert!(backend.is_context_overflow(+ r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#+ ));}#[test]