Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(overflow): add hub GLM phrase to context-overflow detector (#314)

Lin Jia65491db46078
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index 0c5d72854..95de23411 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -24,6 +24,7 @@ const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
"context window",
"context length is only",
"please reduce the length of the input",
+ "exceeds the maximum allowed input length",
];
// Anthropic has no structured `error.code`, so detection is phrase-based only.
@@ -310,6 +311,10 @@ mod tests {
r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
));
assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
+ // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
+ assert!(backend.is_context_overflow(
+ r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
+ ));
}
#[test]
diff --git a/crates/switchyard-components/src/backends/openai.rs b/crates/switchyard-components/src/backends/openai.rs
index 0f9fc7d7d..7cdaf5550 100644
--- a/crates/switchyard-components/src/backends/openai.rs
+++ b/crates/switchyard-components/src/backends/openai.rs
@@ -528,6 +528,7 @@ const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
"context window",
"context length is only",
"please reduce the length of the input",
+ "exceeds the maximum allowed input length",
];
fn is_context_overflow(body: &str) -> bool {
@@ -759,6 +760,14 @@ mod tests {
assert!(!is_context_overflow(body));
}
+ #[test]
+ fn context_overflow_hub_glm_matches() {
+ // Hub GLM error via LiteLLM: code is "400" (not context_length_exceeded),
+ // so detection relies on phrase matching.
+ let body = r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#;
+ assert!(is_context_overflow(body));
+ }
+
#[test]
fn context_overflow_nvidia_litellm_wrap_matches() {
// Body shape observed from inference-api.nvidia.com's LiteLLM proxy