Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): return HTTP 502 for buffered Chat abort, error and repetition finish reasons (#884)
Colin McNamara9a6d26d97dfb
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex b8f5bffe4..8f0793506 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -342,6 +342,23 @@ async fn upstream_chat(.into_response();}+ // A generation that did not complete (`abort` from vLLM or SGLang, `error` from OpenRouter,+ // `repetition` from vLLM).+ if let Some(reason @ ("abort" | "error" | "repetition")) = model.strip_prefix("model/finish-") {+ return Json(json!({+ "id": "chatcmpl-finish",+ "object": "chat.completion",+ "model": model,+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "partial"},+ "finish_reason": reason+ }],+ "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}+ }))+ .into_response();+ }+// Buffered tool call, the non-streaming counterpart of the branch above.if prompt == "mcp-tool-call" {let called = body["tool_choice"]["function"]["name"]@@ -1141,6 +1158,85 @@ async fn failed_responses_return_errors_and_try_fallback_across_endpoints() -> TOk(())}+// A Chat backend that ends a generation with `abort`, `error` or `repetition` did+// not complete it: every client path gets a 502, and a fallback route moves on.+#[tokio::test]+async fn chat_failed_finish_reasons_return_errors_and_try_fallback() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let requests = [+ (+ "/v1/chat/completions",+ json!({+ "model": ROUTE_MODEL, "messages": [{"role": "user", "content": "hello"}]+ }),+ ),+ (+ "/v1/messages",+ json!({+ "model": ROUTE_MODEL, "max_tokens": 16,+ "messages": [{"role": "user", "content": "hello"}]+ }),+ ),+ (+ "/v1/responses",+ json!({"model": ROUTE_MODEL, "input": "hello"}),+ ),+ ];+ for reason in ["abort", "error", "repetition"] {+ let model = format!("model/finish-{reason}");+ let message = format!("provider finished with finish_reason \"{reason}\"");+ for fallback in [false, true] {+ let route = if fallback {+ "type = \"random\"\ntargets = [\"first\", \"second\"]\nweights = [1000, 1]\nseed = 17"+ } else {+ "type = \"passthrough\"\ntarget = \"first\""+ };+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.mock]+format = "openai_chat"+base_url = "{base_url}"+max_retries = 0+[targets]+first = {{ id = "{model}", llm_client = "mock" }}+second = {{ id = "model/fallback", llm_client = "mock" }}+[routes.chat]+id = "{ROUTE_MODEL}"+{route}+"#,+ base_url = upstream.base_url,+ ))?);+ for (path, body) in &requests {+ let previous_calls = upstream.models().await.len();+ let response = send(&app, "POST", path, Some(body.clone())).await?;+ let calls = upstream.models().await[previous_calls..].to_vec();+ if fallback {+ assert_eq!(response.status, StatusCode::OK, "{model}: {path}");+ assert_eq!(+ response.json()?["model"],+ "model/fallback",+ "{model}: {path}"+ );+ assert_eq!(calls, [model.as_str(), "model/fallback"], "{model}: {path}");+ continue;+ }+ assert_eq!(response.status, StatusCode::BAD_GATEWAY, "{model}: {path}");+ let expected = if *path == "/v1/messages" {+ json!({"type": "error", "error": {"type": "api_error", "message": message}})+ } else {+ json!({"error": {+ "type": "upstream_error", "code": "upstream_error", "message": message+ }})+ };+ assert_eq!(response.json()?, expected, "{model}: {path}");+ assert_eq!(calls, [model.as_str()], "{model}: {path}");+ }+ }+ }+ Ok(())+}+#[tokio::test]async fn stats_reset_returns_confirmation_and_clears_all_stats() -> TestResult {let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex 5c36c6d55..f2e66d699 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -274,6 +274,18 @@ impl FormatCodec for OpenAiChatCodec {_policy: &TranslationPolicy,) -> Result<DecodedResponse> {let object = object(body, "$")?;+ // `abort`, `error` and `repetition` mean the generation did not complete.+ if let Some(reason @ ("abort" | "error" | "repetition")) = object+ .get("choices")+ .and_then(Value::as_array)+ .and_then(|choices| choices.first())+ .and_then(|choice| choice.get("finish_reason"))+ .and_then(Value::as_str)+ {+ return Err(TranslationError::UpstreamFailure {+ error: json!({ "message": format!("provider finished with finish_reason \"{reason}\"") }),+ });+ }let mut response = AggLlmResponse {id: object.get("id")diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rsindex 4280fa3f1..20e6dfb30 100644--- a/crates/switchyard-translation/tests/response_translation.rs+++ b/crates/switchyard-translation/tests/response_translation.rs@@ -1057,6 +1057,31 @@ fn failed_responses_return_upstream_failure_with_provider_message() -> TestResulOk(())}+// Verifies Chat `abort`, `error` and `repetition` finish reasons fail the turn.+#[test]+fn failed_finish_reasons_return_upstream_failure() -> TestResult {+ let engine = TranslationEngine::default();+ for reason in ["abort", "error", "repetition"] {+ let body = json!({+ "id": "chatcmpl-test", "object": "chat.completion", "model": "gpt-4o",+ "choices": [{"index": 0, "finish_reason": reason,+ "message": {"role": "assistant", "content": "partial"}}]+ });+ let error = engine+ .translate_response(+ WireFormat::OpenAiChat,+ WireFormat::AnthropicMessages,+ &body,+ &TranslationPolicy::default(),+ )+ .err()+ .ok_or_else(|| format!("accepted finish_reason {reason}"))?;+ assert_eq!(error.kind(), "UpstreamFailure");+ assert!(error.to_string().contains(reason), "{error}");+ }+ Ok(())+}+// Verifies a moderation stop stays distinguishable from a normal turn in both// directions, and that a named refusal category survives re-encoding.#[test]