Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: treat llama.cpp exceed-context errors as overflow (#939)
Dmitry Voropaeva0ece874fd1e
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex 0a28ef6ce..60b962a07 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -32,6 +32,12 @@ const OPENAI_OVERFLOW_PHRASES: &[&str] = &["exceeds the maximum allowed input length","exceeds the maximum allowed length","is longer than the model's context length",+ // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages+ // (tools/server/server-context.cpp). The structured check below catches+ // them from `error.type`; these cover a proxy that rewrites the envelope+ // and keeps the message.+ "exceeds the available context size",+ "larger than the max context size",];// Anthropic has no structured `error.code`, so detection is phrase-based only.@@ -357,11 +363,19 @@ impl Backend {Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(body,|value| {- value- .get("error")- .and_then(|err| err.get("code"))- .and_then(serde_json::Value::as_str)- == Some("context_length_exceeded")+ let error = value.get("error");+ let field = |name| {+ error+ .and_then(|err| err.get(name))+ .and_then(serde_json::Value::as_str)+ };+ // llama-server puts the HTTP status in `code` as a number and+ // names the reason in `type`, so the canonical code never+ // matches. Its two overflow messages are worded unlike any+ // other provider's, which leaves the type as the signal that+ // holds across releases.+ field("code") == Some("context_length_exceeded")+ || field("type") == Some("exceed_context_size_error")},OPENAI_OVERFLOW_PHRASES,),@@ -601,6 +615,31 @@ mod tests {));}+ #[test]+ fn openai_detects_llama_server_overflow() {+ let backend = Backend::OpenAiChat(config("x"));+ // llama-server, numeric `code` and the reason in `type`. Both of the+ // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.+ assert!(backend.is_context_overflow(+ r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#+ ));+ assert!(backend.is_context_overflow(+ r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#+ ));+ // The type alone is enough, so a reworded message still falls back.+ assert!(backend.is_context_overflow(+ r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#+ ));+ // And the message alone is enough, for a proxy that drops the type.+ assert!(backend.is_context_overflow(+ r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#+ ));+ // llama-server's other 400s must still surface to the caller.+ assert!(!backend.is_context_overflow(+ r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#+ ));+ }+#[test]fn anthropic_detects_prompt_too_long() {let backend = Backend::Anthropic(config("x"));