Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy-llm-client): Correctly classify policy-violation (#743)
Graham Kingd59112cd47bb
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex a96a3e1fa..2d8852ac1 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -437,7 +437,7 @@ mod tests {assert!(backend.is_context_overflow(r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#));- // Native SGLang: top-level envelope (no `error` key), caught by the raw-body phrase match.+ // Native SGLang: top-level error envelope (no `error` key).// KV-pool rejection (managers/utils.py) and declared-context rejection// (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.assert!(backend.is_context_overflow(diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex dbb69b2ca..8b13dfd30 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -2525,6 +2525,42 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn content_policy_400_preserves_upstream_error()+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let body = json!({+ "error": {+ "code": "content_policy_violation",+ "message": "request blocked by content policy",+ "type": "invalid_request_error"+ },+ "metadata": {"documentation_section": "context window"}+ });+ Mock::given(method("POST"))+ .respond_with(ResponseTemplate::new(400).set_body_json(&body))+ .expect(1)+ .mount(&server)+ .await;+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let Err(error) = client+ .call_rewrite_model(request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected a policy denial");+ };+ let LlmClientError::UpstreamHttp {+ status,+ body: actual,+ } = error+ else {+ panic!("expected the upstream HTTP error, got {error:?}");+ };+ assert_eq!(status, StatusCode::BAD_REQUEST);+ assert_eq!(serde_json::from_str::<serde_json::Value>(&actual)?, body);+ Ok(())+ }+#[tokio::test]async fn forwards_only_allowlisted_metadata_headers()-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rsindex 0a1219da5..04d0935a8 100644--- a/crates/libsy-llm-client/src/error.rs+++ b/crates/libsy-llm-client/src/error.rs@@ -17,10 +17,9 @@ pub type Result<T> = std::result::Result<T, LlmClientError>;/// and a substring phrase list.////// Parses the body once, runs the structured check (e.g. against `error.code`),-/// then falls back to matching phrases against `error.message` or — when the-/// body is not JSON — the raw body. Centralizing the shape means each new-/// provider-wrap of the canonical error is a one-line phrase entry, not a fork-/// of the parsing logic.+/// then falls back to matching phrases against `error.message` (or `message`+/// for a top-level `object: "error"`) or, when the body is not JSON, the raw body.+/// Other JSON fields do not contribute to overflow detection.pub(crate) fn is_overflow_body<F>(body: &str, structured_check: F, phrases: &[&str]) -> boolwhereF: Fn(&Value) -> bool,@@ -29,14 +28,13 @@ whereif structured_check(&value) {return true;}- if let Some(message) = value- .get("error")+ let error = value.get("error").or_else(|| {+ (value.get("object").and_then(Value::as_str) == Some("error")).then_some(&value)+ });+ return error.and_then(|err| err.get("message")).and_then(Value::as_str)- && contains_any(message, phrases)- {- return true;- }+ .is_some_and(|message| contains_any(message, phrases));}// Some upstream proxies return plain-text bodies; fall through to a string// match on the raw body.diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 0a3408bd7..ef8324578 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -368,6 +368,16 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {match source {LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),LlmClientError::Transport { .. } => Some(RoutingFallbackReason::Unavailable),+ // A policy denial can be specific to one provider. Preserve its HTTP+ // error, but allow another candidate to serve the request.+ LlmClientError::UpstreamHttp { status, body }+ if *status == StatusCode::BAD_REQUEST+ && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {+ value["error"]["code"].as_str() == Some("content_policy_violation")+ }) =>+ {+ Some(RoutingFallbackReason::Unavailable)+ }LlmClientError::UpstreamHttp { status, .. }if matches!(*status,@@ -763,6 +773,7 @@ mod tests {#[derive(Clone, Copy)]enum FirstOutcome {ContextWindow,+ ContentPolicy,Unauthorized,StreamSuccess,MidStreamError,@@ -786,6 +797,10 @@ mod tests {model,message: "too long".to_string(),}),+ FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {+ status: StatusCode::BAD_REQUEST,+ body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),+ }),FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {status: StatusCode::UNAUTHORIZED,body: "unauthorized".to_string(),@@ -1252,7 +1267,7 @@ mod tests {}#[test]- fn fallback_only_accepts_context_and_unavailable_failures() {+ fn fallback_only_accepts_context_policy_and_unavailable_failures() {let error = |source| LibsyError::client_call("target", source);assert_eq!(fallback_reason(&error(LlmClientError::ContextWindowExceeded {@@ -1276,6 +1291,27 @@ mod tests {Some(RoutingFallbackReason::Unavailable));}+ for (body, expected) in [+ (+ r#"{"error":{"code":"content_policy_violation"}}"#,+ Some(RoutingFallbackReason::Unavailable),+ ),+ (+ r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,+ None,+ ),+ (r#"{"error":{"message":"content_policy_violation"}}"#, None),+ ("content_policy_violation", None),+ ] {+ assert_eq!(+ fallback_reason(&error(LlmClientError::UpstreamHttp {+ status: StatusCode::BAD_REQUEST,+ body: body.to_string(),+ })),+ expected,+ "{body}"+ );+ }for status in [StatusCode::BAD_REQUEST,StatusCode::UNAUTHORIZED,@@ -1312,6 +1348,15 @@ mod tests {);assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));+ // A provider-specific policy denial also permits another candidate.+ let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;+ let (_, response) = result?;+ assert_eq!(+ &*client.calls.lock(),+ &[ModelId::from("weak"), "strong".into()]+ );+ assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));+// Authentication failure is not retryable, so the second candidate is untouched.let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;assert!(matches!(