Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): return HTTP 502 for buffered Chat abort, error and repetition finish reasons (#884)

Colin McNamara9a6d26d97dfb
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index b8f5bffe4..8f0793506 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -342,6 +342,23 @@ async fn upstream_chat(
.into_response();
}
+ // A generation that did not complete (`abort` from vLLM or SGLang, `error` from OpenRouter,
+ // `repetition` from vLLM).
+ if let Some(reason @ ("abort" | "error" | "repetition")) = model.strip_prefix("model/finish-") {
+ return Json(json!({
+ "id": "chatcmpl-finish",
+ "object": "chat.completion",
+ "model": model,
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "partial"},
+ "finish_reason": reason
+ }],
+ "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}
+ }))
+ .into_response();
+ }
+
// Buffered tool call, the non-streaming counterpart of the branch above.
if prompt == "mcp-tool-call" {
let called = body["tool_choice"]["function"]["name"]
@@ -1141,6 +1158,85 @@ async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> T
Ok(())
}
+// A Chat backend that ends a generation with `abort`, `error` or `repetition` did
+// not complete it: every client path gets a 502, and a fallback route moves on.
+#[tokio::test]
+async fn chat_failed_finish_reasons_return_errors_and_try_fallback() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let requests = [
+ (
+ "/v1/chat/completions",
+ json!({
+ "model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]
+ }),
+ ),
+ (
+ "/v1/messages",
+ json!({
+ "model": ROUTE_MODEL, "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hello"}]
+ }),
+ ),
+ (
+ "/v1/responses",
+ json!({"model": ROUTE_MODEL, "input": "hello"}),
+ ),
+ ];
+ for reason in ["abort", "error", "repetition"] {
+ let model = format!("model/finish-{reason}");
+ let message = format!("provider finished with finish_reason \"{reason}\"");
+ for fallback in [false, true] {
+ let route = if fallback {
+ "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"
+ } else {
+ "type = \"passthrough\"\ntarget = \"first\""
+ };
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.mock]
+format = "openai_chat"
+base_url = "{base_url}"
+max_retries = 0
+[targets]
+first = {{ id = "{model}", llm_client = "mock" }}
+second = {{ id = "model/fallback", llm_client = "mock" }}
+[routes.chat]
+id = "{ROUTE_MODEL}"
+{route}
+"#,
+ base_url = upstream.base_url,
+ ))?);
+ for (path, body) in &requests {
+ let previous_calls = upstream.models().await.len();
+ let response = send(&app, "POST", path, Some(body.clone())).await?;
+ let calls = upstream.models().await[previous_calls..].to_vec();
+ if fallback {
+ assert_eq!(response.status, StatusCode::OK, "{model}: {path}");
+ assert_eq!(
+ response.json()?["model"],
+ "model/fallback",
+ "{model}: {path}"
+ );
+ assert_eq!(calls, [model.as_str(), "model/fallback"], "{model}: {path}");
+ continue;
+ }
+ assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");
+ let expected = if *path == "/v1/messages" {
+ json!({"type": "error", "error": {"type": "api_error", "message": message}})
+ } else {
+ json!({"error": {
+ "type": "upstream_error", "code": "upstream_error", "message": message
+ }})
+ };
+ assert_eq!(response.json()?, expected, "{model}: {path}");
+ assert_eq!(calls, [model.as_str()], "{model}: {path}");
+ }
+ }
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index 5c36c6d55..f2e66d699 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -274,6 +274,18 @@ impl FormatCodec for OpenAiChatCodec {
_policy: &TranslationPolicy,
) -> Result<DecodedResponse> {
let object = object(body, "$")?;
+ // `abort`, `error` and `repetition` mean the generation did not complete.
+ if let Some(reason @ ("abort" | "error" | "repetition")) = object
+ .get("choices")
+ .and_then(Value::as_array)
+ .and_then(|choices| choices.first())
+ .and_then(|choice| choice.get("finish_reason"))
+ .and_then(Value::as_str)
+ {
+ return Err(TranslationError::UpstreamFailure {
+ error: json!({ "message": format!("provider finished with finish_reason \"{reason}\"") }),
+ });
+ }
let mut response = AggLlmResponse {
id: object
.get("id")
diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rs
index 4280fa3f1..20e6dfb30 100644
--- a/crates/switchyard-translation/tests/response_translation.rs
+++ b/crates/switchyard-translation/tests/response_translation.rs
@@ -1057,6 +1057,31 @@ fn failed_responses_return_upstream_failure_with_provider_message() -> TestResul
Ok(())
}
+// Verifies Chat `abort`, `error` and `repetition` finish reasons fail the turn.
+#[test]
+fn failed_finish_reasons_return_upstream_failure() -> TestResult {
+ let engine = TranslationEngine::default();
+ for reason in ["abort", "error", "repetition"] {
+ let body = json!({
+ "id": "chatcmpl-test", "object": "chat.completion", "model": "gpt-4o",
+ "choices": [{"index": 0, "finish_reason": reason,
+ "message": {"role": "assistant", "content": "partial"}}]
+ });
+ let error = engine
+ .translate_response(
+ WireFormat::OpenAiChat,
+ WireFormat::AnthropicMessages,
+ &body,
+ &TranslationPolicy::default(),
+ )
+ .err()
+ .ok_or_else(|| format!("accepted finish_reason {reason}"))?;
+ assert_eq!(error.kind(), "UpstreamFailure");
+ assert!(error.to_string().contains(reason), "{error}");
+ }
+ Ok(())
+}
+
// Verifies a moderation stop stays distinguishable from a normal turn in both
// directions, and that a named refusal category survives re-encoding.
#[test]