Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): return HTTP 502 when Responses generation fails (#703)
Elyas Mehtabuddin1d1e16fea9d7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9a20cbba7..ed16e6d1e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -121,6 +121,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).### Fixed+- **Return HTTP 502 for failed Responses generations** — a provider's HTTP 200+ response with `status: "failed"` could appear as an empty successful answer.+ Switchyard now counts the call as an error and tries another model when the+ route supports fallback. If this failure reaches the application, Chat+ Completions, Anthropic Messages, and Responses return HTTP 502. Error replies+ preserve the provider's message and, for OpenAI applications, a nonempty+ string error code. Switchyard removes echoed caller credentials.- **Advisor stall checkpoint re-arms after a refunded review** — astall-triggered consult that failed open or returned an unparseable verdictrefunded the review budget but left the conversation's stall latch set, sodiff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex a4511df84..9d2acd16d 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -13,13 +13,13 @@ use futures_util::{StreamExt, stream};use http::StatusCode;use reqwest::RequestBuilder;use reqwest::header::{HeaderMap, RETRY_AFTER};-use serde_json::{Map, Value};+use serde_json::{Map, Value, json};use switchyard_protocol::{LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,Response, RoutedLlmClient,};use switchyard_translation::{- WireFormat, decode_aggregated_response, decode_request, decode_stream,+ TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream,encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,};use tracing::Instrument;@@ -503,8 +503,24 @@ impl TranslatingLlmClient {let body = serde_json::from_slice::<Value>(&body).map_err(|error| {LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))})?;- let agg = decode_aggregated_response(&body, wire_format)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ // Map a provider's failed generation to 502, even under HTTP 200.+ // Redact forwarded credentials before returning the provider error.+ let agg =+ decode_aggregated_response(&body, wire_format).map_err(+ |error| match error {+ TranslationError::UpstreamFailure { error } => {+ LlmClientError::UpstreamHttp {+ status: StatusCode::BAD_GATEWAY,+ body: redact_forwarded_headers(+ json!({ "error": error }).to_string(),+ metadata.as_ref(),+ backend.is_forwarding_auth(),+ ),+ }+ }+ error => LlmClientError::ResponseTranslation(error.to_string()),+ },+ )?;(LlmResponse::Agg(agg), upstream_headers)}};diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex d239d66ad..235329b76 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -1251,12 +1251,7 @@ fn client_error(error: &LlmClientError) -> Response {"invalid_request_error","context_length_exceeded",),- LlmClientError::UpstreamHttp { status, body } => error_response(- *status,- upstream_error_message(body),- "upstream_error",- "upstream_error",- ),+ LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body),LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => {error_response(StatusCode::BAD_GATEWAY,@@ -1282,26 +1277,25 @@ fn client_error(error: &LlmClientError) -> Response {}}-// Provider errors are often JSON documents; expose their message without-// embedding the entire document as an escaped string in our error envelope.-fn upstream_error_message(body: &str) -> String {- serde_json::from_str::<Value>(body)- .ok()- .and_then(|body| {- body.pointer("/error/message")- .and_then(Value::as_str)- .map(str::to_string)- })- .unwrap_or_else(|| body.to_string())+// Keep the provider's message and nonempty string code in our error JSON.+fn upstream_error(status: StatusCode, body: &str) -> Response {+ let parsed = serde_json::from_str::<Value>(body).unwrap_or_default();+ let error = &parsed["error"];+ let message = error["message"].as_str().unwrap_or(body);+ let code = error["code"]+ .as_str()+ .filter(|code| !code.is_empty())+ .unwrap_or("upstream_error");+ error_response(status, message, "upstream_error", code)}-// Error metadata retained until the client-facing endpoint selects an envelope.+// Keep error details until the endpoint chooses its response format.#[derive(Clone)]struct ApiError {status: StatusCode,message: String,error_type: &'static str,- code: &'static str,+ code: String,}impl ApiError {@@ -1309,13 +1303,13 @@ impl ApiError {status: StatusCode,message: impl Into<String>,error_type: &'static str,- code: &'static str,+ code: impl Into<String>,) -> Self {Self {status,message: message.into(),error_type,- code,+ code: code.into(),}}@@ -1386,7 +1380,7 @@ fn error_response(status: StatusCode,message: impl Into<String>,error_type: &'static str,- code: &'static str,+ code: impl Into<String>,) -> Response {ApiError::new(status, message, error_type, code).into_response(WireFormat::OpenAiChat)}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 7097a7529..11cd3e9f9 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -51,6 +51,7 @@ impl MockUpstream {let calls = Arc::new(Mutex::new(Vec::new()));let app = Router::new().route("/v1/chat/completions", post(upstream_chat))+ .route("/buffered/responses", post(upstream_buffered_responses)).route("/v1/messages",post(upstream_messages_requires_forwarded_oauth),@@ -521,6 +522,43 @@ async fn upstream_responses_requires_forwarded_auth(.into_response()}+async fn upstream_buffered_responses(+ State(calls): State<Arc<Mutex<Vec<Value>>>>,+ headers: HeaderMap,+ Json(body): Json<Value>,+) -> Json<Value> {+ calls.lock().await.push(body.clone());+ let model = body["model"].as_str().unwrap_or_default();+ let mut response = json!({+ "id": "resp_buffered", "object": "response", "model": model,+ "status": "failed", "output": [], "usage": null,+ "error": {"code": "server_error", "message": "deterministic upstream failure"}+ });+ match model {+ "model/missing-error" => response = json!({"status": "failed"}),+ "model/invalid-error" => response["error"] = json!("invalid error details"),+ "model/invalid-code" => response["error"]["code"] = json!(42),+ "model/empty-code" => response["error"]["code"] = json!(""),+ "model/fallback" => {+ response["status"] = json!("completed");+ response["error"] = Value::Null;+ response["output"] = json!([{+ "type": "message", "role": "assistant",+ "content": [{"type": "output_text", "text": "ok"}]+ }]);+ }+ _ => {}+ }+ if let Some(secret) = headers.get("x-private-token") {+ response["error"]["code"] = json!(secret.to_str().unwrap_or_default());+ response["error"]["message"] = json!(format!(+ "provider rejected {}",+ secret.to_str().unwrap_or_default()+ ));+ }+ Json(response)+}+async fn upstream_redirect_capture(State(calls): State<Arc<Mutex<Vec<Value>>>>,headers: HeaderMap,@@ -803,6 +841,120 @@ async fn stats_accumulates_buffered_success_error_and_shared_routes() -> TestResOk(())}+fn buffered_responses_app(+ upstream: &MockUpstream,+ model: &str,+ fallback: bool,+ forward_auth: bool,+) -> TestResult<Router> {+ let route = if fallback {+ "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"+ } else {+ "type = \"passthrough\"\ntarget = \"first\""+ };+ let state = load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.mock]+format = "openai_responses"+base_url = "{base_url}/buffered"+forward_auth = {forward_auth}+max_retries = 0+[targets]+first = {{ id = "{model}", llm_client = "mock" }}+second = {{ id = "model/fallback", llm_client = "mock" }}+[routes.response]+id = "{ROUTE_MODEL}"+{route}+"#,+ base_url = upstream.base_url.trim_end_matches("/v1"),+ ))?;+ Ok(build_switchyard_router(state))+}++#[tokio::test]+async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let requests = [+ (+ "/v1/chat/completions",+ json!({+ "model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]+ }),+ ),+ (+ "/v1/messages",+ json!({+ "model": ROUTE_MODEL, "max_tokens": 16,+ "messages": [{"role": "user", "content": "hello"}]+ }),+ ),+ (+ "/v1/responses",+ json!({"model": ROUTE_MODEL, "input": "hello"}),+ ),+ ];+ let failure = "deterministic upstream failure";+ let missing = "provider reported status \"failed\" without error details";+ for (model, message, code) in [+ ("model/failed", failure, "server_error"),+ ("model/missing-error", missing, "upstream_error"),+ ("model/invalid-error", missing, "upstream_error"),+ ("model/invalid-code", failure, "upstream_error"),+ ("model/empty-code", failure, "upstream_error"),+ ] {+ let app = buffered_responses_app(&upstream, model, false, false)?;+ for (path, body) in &requests {+ let response = send(&app, "POST", path, Some(body.clone())).await?;+ assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");+ let expected = if *path == "/v1/messages" {+ json!({"type": "error", "error": {"type": "api_error", "message": message}})+ } else {+ json!({"error": {"type": "upstream_error", "code": code, "message": message}})+ };+ assert_eq!(response.json()?, expected, "{model}: {path}");+ }+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["total_requests"], requests.len(), "{model}");+ assert_eq!(stats["total_errors"], requests.len(), "{model}");+ assert_eq!(stats["models"][model]["errors"], requests.len(), "{model}");+ }++ let app = buffered_responses_app(&upstream, "model/failed", false, true)?;+ let (path, body) = &requests[0];+ let response = send_with_headers(+ &app,+ "POST",+ path,+ Some(body.clone()),+ &[("x-private-token", "test-private-credential")],+ )+ .await?;+ assert_eq!(response.status, StatusCode::BAD_GATEWAY);+ assert_eq!(+ response.json()?["error"]["message"],+ "provider rejected [REDACTED]"+ );+ assert_eq!(response.json()?["error"]["code"], "[REDACTED]");++ let app = buffered_responses_app(&upstream, "model/failed", true, false)?;+ for (path, body) in &requests {+ let previous_calls = upstream.models().await.len();+ let response = send(&app, "POST", path, Some(body.clone())).await?;+ assert_eq!(response.status, StatusCode::OK, "{path}");+ assert_eq!(response.json()?["model"], "model/fallback", "{path}");+ assert_eq!(+ &upstream.models().await[previous_calls..],+ ["model/failed", "model/fallback"],+ "{path}"+ );+ }+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["models"]["model/failed"]["errors"], requests.len());+ assert_eq!(stats["models"]["model/fallback"]["calls"], requests.len());+ Ok(())+}+#[tokio::test]async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 4b4a2671d..8c6a3c078 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -262,6 +262,19 @@ impl FormatCodec for OpenAiResponsesCodec {fn decode_response(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedResponse> {let body = crate::util::object(body, "$")?;+ // Providers can return HTTP 200 when generation fails. Check `status`+ // before treating an empty `output` as a completed turn.+ if body.get("status").and_then(Value::as_str) == Some("failed") {+ return Err(TranslationError::UpstreamFailure {+ error: body+ .get("error")+ .filter(|error| error.is_object())+ .cloned()+ .unwrap_or_else(|| {+ json!({ "message": "provider reported status \"failed\" without error details" })+ }),+ });+ }let mut diagnostics = Vec::new();let mut content = Vec::new();let mut role = Role::Assistant;diff --git a/crates/switchyard-translation/src/error.rs b/crates/switchyard-translation/src/error.rsindex 80ced348d..56c71d4c6 100644--- a/crates/switchyard-translation/src/error.rs+++ b/crates/switchyard-translation/src/error.rs@@ -34,6 +34,11 @@ pub enum TranslationError {#[error("invalid value at {path}: {message}")]InvalidValue { path: String, message: String },+ /// The response body reports a failed generation, such as OpenAI Responses+ /// `status: "failed"`. `error` contains the provider's error object or a+ /// message explaining that the provider supplied no error details.+ #[error("upstream reported a failed response: {error}")]+ UpstreamFailure { error: serde_json::Value },#[error("{0}")]Other(String),}@@ -48,6 +53,7 @@ impl TranslationError {Self::LossyConversion(_) => "LossyConversion",Self::UnknownField { .. } => "UnknownField",Self::InvalidValue { .. } => "InvalidValue",+ Self::UpstreamFailure { .. } => "UpstreamFailure",Self::Other(_) => "Other",}}diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rsindex e68a1ad61..73b885462 100644--- a/crates/switchyard-translation/tests/response_translation.rs+++ b/crates/switchyard-translation/tests/response_translation.rs@@ -651,6 +651,34 @@ fn incomplete_responses_source_survives_translation() -> TestResult {Ok(())}+#[test]+fn failed_responses_return_upstream_failure_with_provider_message() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "id": "resp_failed",+ "object": "response",+ "model": "gpt-4o",+ "status": "failed",+ "error": {"code": "server_error", "message": "deterministic upstream failure"},+ "output": [],+ "usage": null+ });++ let target = WireFormat::OpenAiChat;+ let error = engine+ .translate_response(+ WireFormat::OpenAiResponses,+ target,+ &body,+ &TranslationPolicy::default(),+ )+ .err()+ .ok_or_else(|| format!("{target:?} accepted a failed response"))?;+ assert_eq!(error.kind(), "UpstreamFailure");+ assert!(error.to_string().contains("deterministic upstream failure"));+ Ok(())+}+// Verifies a moderation stop stays distinguishable from a normal turn in both// directions, and that a named refusal category survives re-encoding.#[test]