Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): return HTTP 502 when Responses generation fails (#703)

Elyas Mehtabuddin1d1e16fea9d7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a20cbba7..ed16e6d1e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -121,6 +121,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Fixed
+- **Return HTTP 502 for failed Responses generations** — a provider's HTTP 200
+ response with `status: "failed"` could appear as an empty successful answer.
+ Switchyard now counts the call as an error and tries another model when the
+ route supports fallback. If this failure reaches the application, Chat
+ Completions, Anthropic Messages, and Responses return HTTP 502. Error replies
+ preserve the provider's message and, for OpenAI applications, a nonempty
+ string error code. Switchyard removes echoed caller credentials.
- **Advisor stall checkpoint re-arms after a refunded review** — a
stall-triggered consult that failed open or returned an unparseable verdict
refunded the review budget but left the conversation's stall latch set, so
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index a4511df84..9d2acd16d 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -13,13 +13,13 @@ use futures_util::{StreamExt, stream};
use http::StatusCode;
use reqwest::RequestBuilder;
use reqwest::header::{HeaderMap, RETRY_AFTER};
-use serde_json::{Map, Value};
+use serde_json::{Map, Value, json};
use switchyard_protocol::{
LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,
Response, RoutedLlmClient,
};
use switchyard_translation::{
- WireFormat, decode_aggregated_response, decode_request, decode_stream,
+ TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream,
encode_aggregated_response_with_extensions, encode_request, encode_stream_with_extensions,
};
use tracing::Instrument;
@@ -503,8 +503,24 @@ impl TranslatingLlmClient {
let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
})?;
- let agg = decode_aggregated_response(&body, wire_format)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ // Map a provider's failed generation to 502, even under HTTP 200.
+ // Redact forwarded credentials before returning the provider error.
+ let agg =
+ decode_aggregated_response(&body, wire_format).map_err(
+ |error| match error {
+ TranslationError::UpstreamFailure { error } => {
+ LlmClientError::UpstreamHttp {
+ status: StatusCode::BAD_GATEWAY,
+ body: redact_forwarded_headers(
+ json!({ "error": error }).to_string(),
+ metadata.as_ref(),
+ backend.is_forwarding_auth(),
+ ),
+ }
+ }
+ error => LlmClientError::ResponseTranslation(error.to_string()),
+ },
+ )?;
(LlmResponse::Agg(agg), upstream_headers)
}
};
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index d239d66ad..235329b76 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -1251,12 +1251,7 @@ fn client_error(error: &LlmClientError) -> Response {
"invalid_request_error",
"context_length_exceeded",
),
- LlmClientError::UpstreamHttp { status, body } => error_response(
- *status,
- upstream_error_message(body),
- "upstream_error",
- "upstream_error",
- ),
+ LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body),
LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => {
error_response(
StatusCode::BAD_GATEWAY,
@@ -1282,26 +1277,25 @@ fn client_error(error: &LlmClientError) -> Response {
}
}
-// Provider errors are often JSON documents; expose their message without
-// embedding the entire document as an escaped string in our error envelope.
-fn upstream_error_message(body: &str) -> String {
- serde_json::from_str::<Value>(body)
- .ok()
- .and_then(|body| {
- body.pointer("/error/message")
- .and_then(Value::as_str)
- .map(str::to_string)
- })
- .unwrap_or_else(|| body.to_string())
+// Keep the provider's message and nonempty string code in our error JSON.
+fn upstream_error(status: StatusCode, body: &str) -> Response {
+ let parsed = serde_json::from_str::<Value>(body).unwrap_or_default();
+ let error = &parsed["error"];
+ let message = error["message"].as_str().unwrap_or(body);
+ let code = error["code"]
+ .as_str()
+ .filter(|code| !code.is_empty())
+ .unwrap_or("upstream_error");
+ error_response(status, message, "upstream_error", code)
}
-// Error metadata retained until the client-facing endpoint selects an envelope.
+// Keep error details until the endpoint chooses its response format.
#[derive(Clone)]
struct ApiError {
status: StatusCode,
message: String,
error_type: &'static str,
- code: &'static str,
+ code: String,
}
impl ApiError {
@@ -1309,13 +1303,13 @@ impl ApiError {
status: StatusCode,
message: impl Into<String>,
error_type: &'static str,
- code: &'static str,
+ code: impl Into<String>,
) -> Self {
Self {
status,
message: message.into(),
error_type,
- code,
+ code: code.into(),
}
}
@@ -1386,7 +1380,7 @@ fn error_response(
status: StatusCode,
message: impl Into<String>,
error_type: &'static str,
- code: &'static str,
+ code: impl Into<String>,
) -> Response {
ApiError::new(status, message, error_type, code).into_response(WireFormat::OpenAiChat)
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 7097a7529..11cd3e9f9 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -51,6 +51,7 @@ impl MockUpstream {
let calls = Arc::new(Mutex::new(Vec::new()));
let app = Router::new()
.route("/v1/chat/completions", post(upstream_chat))
+ .route("/buffered/responses", post(upstream_buffered_responses))
.route(
"/v1/messages",
post(upstream_messages_requires_forwarded_oauth),
@@ -521,6 +522,43 @@ async fn upstream_responses_requires_forwarded_auth(
.into_response()
}
+async fn upstream_buffered_responses(
+ State(calls): State<Arc<Mutex<Vec<Value>>>>,
+ headers: HeaderMap,
+ Json(body): Json<Value>,
+) -> Json<Value> {
+ calls.lock().await.push(body.clone());
+ let model = body["model"].as_str().unwrap_or_default();
+ let mut response = json!({
+ "id": "resp_buffered", "object": "response", "model": model,
+ "status": "failed", "output": [], "usage": null,
+ "error": {"code": "server_error", "message": "deterministic upstream failure"}
+ });
+ match model {
+ "model/missing-error" => response = json!({"status": "failed"}),
+ "model/invalid-error" => response["error"] = json!("invalid error details"),
+ "model/invalid-code" => response["error"]["code"] = json!(42),
+ "model/empty-code" => response["error"]["code"] = json!(""),
+ "model/fallback" => {
+ response["status"] = json!("completed");
+ response["error"] = Value::Null;
+ response["output"] = json!([{
+ "type": "message", "role": "assistant",
+ "content": [{"type": "output_text", "text": "ok"}]
+ }]);
+ }
+ _ => {}
+ }
+ if let Some(secret) = headers.get("x-private-token") {
+ response["error"]["code"] = json!(secret.to_str().unwrap_or_default());
+ response["error"]["message"] = json!(format!(
+ "provider rejected {}",
+ secret.to_str().unwrap_or_default()
+ ));
+ }
+ Json(response)
+}
+
async fn upstream_redirect_capture(
State(calls): State<Arc<Mutex<Vec<Value>>>>,
headers: HeaderMap,
@@ -803,6 +841,120 @@ async fn stats_accumulates_buffered_success_error_and_shared_routes() -> TestRes
Ok(())
}
+fn buffered_responses_app(
+ upstream: &MockUpstream,
+ model: &str,
+ fallback: bool,
+ forward_auth: bool,
+) -> TestResult<Router> {
+ let route = if fallback {
+ "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"
+ } else {
+ "type = \"passthrough\"\ntarget = \"first\""
+ };
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.mock]
+format = "openai_responses"
+base_url = "{base_url}/buffered"
+forward_auth = {forward_auth}
+max_retries = 0
+[targets]
+first = {{ id = "{model}", llm_client = "mock" }}
+second = {{ id = "model/fallback", llm_client = "mock" }}
+[routes.response]
+id = "{ROUTE_MODEL}"
+{route}
+"#,
+ base_url = upstream.base_url.trim_end_matches("/v1"),
+ ))?;
+ Ok(build_switchyard_router(state))
+}
+
+#[tokio::test]
+async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let requests = [
+ (
+ "/v1/chat/completions",
+ json!({
+ "model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]
+ }),
+ ),
+ (
+ "/v1/messages",
+ json!({
+ "model": ROUTE_MODEL, "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hello"}]
+ }),
+ ),
+ (
+ "/v1/responses",
+ json!({"model": ROUTE_MODEL, "input": "hello"}),
+ ),
+ ];
+ let failure = "deterministic upstream failure";
+ let missing = "provider reported status \"failed\" without error details";
+ for (model, message, code) in [
+ ("model/failed", failure, "server_error"),
+ ("model/missing-error", missing, "upstream_error"),
+ ("model/invalid-error", missing, "upstream_error"),
+ ("model/invalid-code", failure, "upstream_error"),
+ ("model/empty-code", failure, "upstream_error"),
+ ] {
+ let app = buffered_responses_app(&upstream, model, false, false)?;
+ for (path, body) in &requests {
+ let response = send(&app, "POST", path, Some(body.clone())).await?;
+ assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");
+ let expected = if *path == "/v1/messages" {
+ json!({"type": "error", "error": {"type": "api_error", "message": message}})
+ } else {
+ json!({"error": {"type": "upstream_error", "code": code, "message": message}})
+ };
+ assert_eq!(response.json()?, expected, "{model}: {path}");
+ }
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["total_requests"], requests.len(), "{model}");
+ assert_eq!(stats["total_errors"], requests.len(), "{model}");
+ assert_eq!(stats["models"][model]["errors"], requests.len(), "{model}");
+ }
+
+ let app = buffered_responses_app(&upstream, "model/failed", false, true)?;
+ let (path, body) = &requests[0];
+ let response = send_with_headers(
+ &app,
+ "POST",
+ path,
+ Some(body.clone()),
+ &[("x-private-token", "test-private-credential")],
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::BAD_GATEWAY);
+ assert_eq!(
+ response.json()?["error"]["message"],
+ "provider rejected [REDACTED]"
+ );
+ assert_eq!(response.json()?["error"]["code"], "[REDACTED]");
+
+ let app = buffered_responses_app(&upstream, "model/failed", true, false)?;
+ for (path, body) in &requests {
+ let previous_calls = upstream.models().await.len();
+ let response = send(&app, "POST", path, Some(body.clone())).await?;
+ assert_eq!(response.status, StatusCode::OK, "{path}");
+ assert_eq!(response.json()?["model"], "model/fallback", "{path}");
+ assert_eq!(
+ &upstream.models().await[previous_calls..],
+ ["model/failed", "model/fallback"],
+ "{path}"
+ );
+ }
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["models"]["model/failed"]["errors"], requests.len());
+ assert_eq!(stats["models"]["model/fallback"]["calls"], requests.len());
+ Ok(())
+}
+
#[tokio::test]
async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 4b4a2671d..8c6a3c078 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -262,6 +262,19 @@ impl FormatCodec for OpenAiResponsesCodec {
fn decode_response(&self, body: &Value, policy: &TranslationPolicy) -> Result<DecodedResponse> {
let body = crate::util::object(body, "$")?;
+ // Providers can return HTTP 200 when generation fails. Check `status`
+ // before treating an empty `output` as a completed turn.
+ if body.get("status").and_then(Value::as_str) == Some("failed") {
+ return Err(TranslationError::UpstreamFailure {
+ error: body
+ .get("error")
+ .filter(|error| error.is_object())
+ .cloned()
+ .unwrap_or_else(|| {
+ json!({ "message": "provider reported status \"failed\" without error details" })
+ }),
+ });
+ }
let mut diagnostics = Vec::new();
let mut content = Vec::new();
let mut role = Role::Assistant;
diff --git a/crates/switchyard-translation/src/error.rs b/crates/switchyard-translation/src/error.rs
index 80ced348d..56c71d4c6 100644
--- a/crates/switchyard-translation/src/error.rs
+++ b/crates/switchyard-translation/src/error.rs
@@ -34,6 +34,11 @@ pub enum TranslationError {
#[error("invalid value at {path}: {message}")]
InvalidValue { path: String, message: String },
+ /// The response body reports a failed generation, such as OpenAI Responses
+ /// `status: "failed"`. `error` contains the provider's error object or a
+ /// message explaining that the provider supplied no error details.
+ #[error("upstream reported a failed response: {error}")]
+ UpstreamFailure { error: serde_json::Value },
#[error("{0}")]
Other(String),
}
@@ -48,6 +53,7 @@ impl TranslationError {
Self::LossyConversion(_) => "LossyConversion",
Self::UnknownField { .. } => "UnknownField",
Self::InvalidValue { .. } => "InvalidValue",
+ Self::UpstreamFailure { .. } => "UpstreamFailure",
Self::Other(_) => "Other",
}
}
diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs
index e68a1ad61..73b885462 100644
--- a/crates/switchyard-translation/tests/response_translation.rs
+++ b/crates/switchyard-translation/tests/response_translation.rs
@@ -651,6 +651,34 @@ fn incomplete_responses_source_survives_translation() -> TestResult {
Ok(())
}
+#[test]
+fn failed_responses_return_upstream_failure_with_provider_message() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "id": "resp_failed",
+ "object": "response",
+ "model": "gpt-4o",
+ "status": "failed",
+ "error": {"code": "server_error", "message": "deterministic upstream failure"},
+ "output": [],
+ "usage": null
+ });
+
+ let target = WireFormat::OpenAiChat;
+ let error = engine
+ .translate_response(
+ WireFormat::OpenAiResponses,
+ target,
+ &body,
+ &TranslationPolicy::default(),
+ )
+ .err()
+ .ok_or_else(|| format!("{target:?} accepted a failed response"))?;
+ assert_eq!(error.kind(), "UpstreamFailure");
+ assert!(error.to_string().contains("deterministic upstream failure"));
+ Ok(())
+}
+
// Verifies a moderation stop stays distinguishable from a normal turn in both
// directions, and that a named refusal category survives re-encoding.
#[test]