Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy-llm-client): Correctly classify policy-violation (#743)

Graham Kingd59112cd47bb
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index a96a3e1fa..2d8852ac1 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -437,7 +437,7 @@ mod tests {
assert!(backend.is_context_overflow(
r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
));
- // Native SGLang: top-level envelope (no `error` key), caught by the raw-body phrase match.
+ // Native SGLang: top-level error envelope (no `error` key).
// KV-pool rejection (managers/utils.py) and declared-context rejection
// (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
assert!(backend.is_context_overflow(
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index dbb69b2ca..8b13dfd30 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -2525,6 +2525,42 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn content_policy_400_preserves_upstream_error()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let body = json!({
+ "error": {
+ "code": "content_policy_violation",
+ "message": "request blocked by content policy",
+ "type": "invalid_request_error"
+ },
+ "metadata": {"documentation_section": "context window"}
+ });
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(400).set_body_json(&body))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let Err(error) = client
+ .call_rewrite_model(request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected a policy denial");
+ };
+ let LlmClientError::UpstreamHttp {
+ status,
+ body: actual,
+ } = error
+ else {
+ panic!("expected the upstream HTTP error, got {error:?}");
+ };
+ assert_eq!(status, StatusCode::BAD_REQUEST);
+ assert_eq!(serde_json::from_str::<serde_json::Value>(&actual)?, body);
+ Ok(())
+ }
+
#[tokio::test]
async fn forwards_only_allowlisted_metadata_headers()
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rs
index 0a1219da5..04d0935a8 100644
--- a/crates/libsy-llm-client/src/error.rs
+++ b/crates/libsy-llm-client/src/error.rs
@@ -17,10 +17,9 @@ pub type Result<T> = std::result::Result<T, LlmClientError>;
/// and a substring phrase list.
///
/// Parses the body once, runs the structured check (e.g. against `error.code`),
-/// then falls back to matching phrases against `error.message` or — when the
-/// body is not JSON — the raw body. Centralizing the shape means each new
-/// provider-wrap of the canonical error is a one-line phrase entry, not a fork
-/// of the parsing logic.
+/// then falls back to matching phrases against `error.message` (or `message`
+/// for a top-level `object: "error"`) or, when the body is not JSON, the raw body.
+/// Other JSON fields do not contribute to overflow detection.
pub(crate) fn is_overflow_body<F>(body: &str, structured_check: F, phrases: &[&str]) -> bool
where
F: Fn(&Value) -> bool,
@@ -29,14 +28,13 @@ where
if structured_check(&value) {
return true;
}
- if let Some(message) = value
- .get("error")
+ let error = value.get("error").or_else(|| {
+ (value.get("object").and_then(Value::as_str) == Some("error")).then_some(&value)
+ });
+ return error
.and_then(|err| err.get("message"))
.and_then(Value::as_str)
- && contains_any(message, phrases)
- {
- return true;
- }
+ .is_some_and(|message| contains_any(message, phrases));
}
// Some upstream proxies return plain-text bodies; fall through to a string
// match on the raw body.
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 0a3408bd7..ef8324578 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -368,6 +368,16 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
match source {
LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),
LlmClientError::Transport { .. } => Some(RoutingFallbackReason::Unavailable),
+ // A policy denial can be specific to one provider. Preserve its HTTP
+ // error, but allow another candidate to serve the request.
+ LlmClientError::UpstreamHttp { status, body }
+ if *status == StatusCode::BAD_REQUEST
+ && serde_json::from_str::<serde_json::Value>(body).is_ok_and(|value| {
+ value["error"]["code"].as_str() == Some("content_policy_violation")
+ }) =>
+ {
+ Some(RoutingFallbackReason::Unavailable)
+ }
LlmClientError::UpstreamHttp { status, .. }
if matches!(
*status,
@@ -763,6 +773,7 @@ mod tests {
#[derive(Clone, Copy)]
enum FirstOutcome {
ContextWindow,
+ ContentPolicy,
Unauthorized,
StreamSuccess,
MidStreamError,
@@ -786,6 +797,10 @@ mod tests {
model,
message: "too long".to_string(),
}),
+ FirstOutcome::ContentPolicy => Err(LlmClientError::UpstreamHttp {
+ status: StatusCode::BAD_REQUEST,
+ body: r#"{"error":{"code":"content_policy_violation","message":"request blocked by content policy","type":"invalid_request_error"},"metadata":{"documentation_section":"context window"}}"#.to_string(),
+ }),
FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {
status: StatusCode::UNAUTHORIZED,
body: "unauthorized".to_string(),
@@ -1252,7 +1267,7 @@ mod tests {
}
#[test]
- fn fallback_only_accepts_context_and_unavailable_failures() {
+ fn fallback_only_accepts_context_policy_and_unavailable_failures() {
let error = |source| LibsyError::client_call("target", source);
assert_eq!(
fallback_reason(&error(LlmClientError::ContextWindowExceeded {
@@ -1276,6 +1291,27 @@ mod tests {
Some(RoutingFallbackReason::Unavailable)
);
}
+ for (body, expected) in [
+ (
+ r#"{"error":{"code":"content_policy_violation"}}"#,
+ Some(RoutingFallbackReason::Unavailable),
+ ),
+ (
+ r#"{"error":{"code":"invalid_request_error"},"metadata":"content_policy_violation"}"#,
+ None,
+ ),
+ (r#"{"error":{"message":"content_policy_violation"}}"#, None),
+ ("content_policy_violation", None),
+ ] {
+ assert_eq!(
+ fallback_reason(&error(LlmClientError::UpstreamHttp {
+ status: StatusCode::BAD_REQUEST,
+ body: body.to_string(),
+ })),
+ expected,
+ "{body}"
+ );
+ }
for status in [
StatusCode::BAD_REQUEST,
StatusCode::UNAUTHORIZED,
@@ -1312,6 +1348,15 @@ mod tests {
);
assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
+ // A provider-specific policy denial also permits another candidate.
+ let (client, result) = run_candidates(FirstOutcome::ContentPolicy).await;
+ let (_, response) = result?;
+ assert_eq!(
+ &*client.calls.lock(),
+ &[ModelId::from("weak"), "strong".into()]
+ );
+ assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
+
// Authentication failure is not retryable, so the second candidate is untouched.
let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;
assert!(matches!(