Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): report Responses max-token truncation as incomplete (#300)
ryan-lempka5e83e2b94fc3
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 0bc964e19..5d3b9e888 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -199,6 +199,16 @@ impl FormatCodec for OpenAiResponsesCodec {}}}+ // The truncation signal is on the response, not the output items.+ if body.get("status").and_then(Value::as_str) == Some("incomplete")+ && body+ .get("incomplete_details")+ .and_then(|details| details.get("reason"))+ .and_then(Value::as_str)+ == Some("max_output_tokens")+ {+ stop_reason = Some(StopReason::MaxTokens);+ }let outputs = vec![if has_output {ResponseOutput {role,@@ -256,6 +266,18 @@ impl FormatCodec for OpenAiResponsesCodec {diagnostics: Vec::new(),});}+ let is_truncated = matches!(+ response+ .first_output()+ .and_then(|output| output.stop_reason),+ Some(StopReason::MaxTokens)+ );+ let status = if is_truncated {+ "incomplete"+ } else {+ "completed"+ };+ let incomplete_details = is_truncated.then(|| json!({ "reason": "max_output_tokens" }));Ok(EncodedResponse {body: embed_preservation(json!({@@ -263,7 +285,8 @@ impl FormatCodec for OpenAiResponsesCodec {"object": "response","created_at": 0,"model": response.model.clone().unwrap_or_else(|| "unknown".to_string()),- "status": "completed",+ "status": status,+ "incomplete_details": incomplete_details,"output": encode_responses_output(&response.outputs),"usage": encode_responses_usage(&response.usage),"parallel_tool_calls": true,@@ -1093,6 +1116,11 @@ fn encode_responses_output(outputs: &[ResponseOutput]) -> Value {.any(|block| matches!(block, ContentBlock::ToolCall(_)));let text = text_from_blocks(&output.content, "");let reasoning = reasoning_text_from_blocks(&output.content, "\n");+ let status = if matches!(output.stop_reason, Some(StopReason::MaxTokens)) {+ "incomplete"+ } else {+ "completed"+ };let mut items = Vec::new();if !reasoning.is_empty() {@@ -1103,7 +1131,7 @@ fn encode_responses_output(outputs: &[ResponseOutput]) -> Value {items.push(json!({"type": "message","id": "msg_switchyard",- "status": "completed",+ "status": status,"role": role_to_responses(output.role),"content": [{"type": "output_text",diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 10bf323f3..cd531cb60 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -137,6 +137,10 @@ fn decode_responses_stream(out.push(LlmResponseChunk::MessageStop { reason: None });out}+ // Carries the Anthropic spelling because every encoder already maps it.+ Some("response.incomplete") => vec![LlmResponseChunk::MessageStop {+ reason: Some("max_tokens".to_string()),+ }],Some("error") => vec![LlmResponseChunk::StreamError {message: event.get("message")@@ -188,6 +192,16 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {if state.finished {return Vec::new();}+ let is_truncated = matches!(+ state.stop_reason.as_deref(),+ Some("length") | Some("max_tokens")+ );+ let (event_type, status) = if is_truncated {+ ("response.incomplete", "incomplete")+ } else {+ ("response.completed", "completed")+ };+ let incomplete_details = is_truncated.then(|| json!({ "reason": "max_output_tokens" }));let mut out = ensure_responses_created(state);if state.response_text_started&& let Some(output_index) = state.response_text_output_index@@ -204,7 +218,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {"item": {"type": "message","role": "assistant",- "status": "completed",+ "status": status,"content": [{"type": "output_text", "text": state.response_text}],},}));@@ -245,7 +259,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {json!({"type": "message","role": "assistant",- "status": "completed",+ "status": status,"content": [{"type": "output_text", "text": state.response_text}],}),));@@ -284,11 +298,12 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {.collect::<Vec<_>>();out.push(json!({- "type": "response.completed",+ "type": event_type,"response": {"id": responses_id(state),"object": "response",- "status": "completed",+ "status": status,+ "incomplete_details": incomplete_details,"model": target_model_or_source_model(state),"output": output,"usage": responses_usage_value(&state.usage),diff --git a/crates/switchyard-translation/tests/response_translation.rs b/crates/switchyard-translation/tests/response_translation.rsindex c837fa08b..cc863ece3 100644--- a/crates/switchyard-translation/tests/response_translation.rs+++ b/crates/switchyard-translation/tests/response_translation.rs@@ -536,3 +536,67 @@ fn openai_chat_cache_only_usage_still_emits_reasoning_details() -> TestResult {);Ok(())}++// Verifies a token-limit stop is reported as an incomplete Responses result.+#[test]+fn openai_chat_length_finish_translates_to_incomplete_responses_status() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "id": "chatcmpl-test",+ "model": "gpt-4o",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "Half an ans"},+ "finish_reason": "length"+ }],+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}+ });++ let output = engine+ .translate_response(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(output["status"], "incomplete");+ assert_eq!(+ output["incomplete_details"],+ json!({"reason": "max_output_tokens"})+ );+ assert_eq!(output["output"][0]["status"], "incomplete");+ Ok(())+}++// Verifies a truncated Responses source keeps its stop reason when re-encoded.+#[test]+fn incomplete_responses_source_survives_translation() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "id": "resp_1",+ "object": "response",+ "model": "gpt-4o",+ "status": "incomplete",+ "incomplete_details": {"reason": "max_output_tokens"},+ "output": [{+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "Half an ans"}]+ }],+ "usage": {"input_tokens": 10, "output_tokens": 1, "total_tokens": 11}+ });++ let output = engine+ .translate_response(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(output["choices"][0]["finish_reason"], "length");+ Ok(())+}diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex b4b574a31..5ebd784b0 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -955,3 +955,98 @@ fn openai_chat_stream_usage_without_breakdowns_still_emits_responses_usage_detai);Ok(())}++// Verifies a streamed token-limit stop terminates with response.incomplete.+#[test]+fn openai_chat_length_finish_translates_to_responses_incomplete_event() -> TestResult {+ let engine = TranslationEngine::default();+ let mut state =+ StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses);+ let chunk = json!({+ "id": "chatcmpl-test",+ "object": "chat.completion.chunk",+ "model": "gpt-4o",+ "choices": [{+ "index": 0,+ "delta": {"content": "Half an ans"},+ "finish_reason": "length"+ }]+ });++ let mut events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ &chunk,+ )?;+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);++ let Some(terminal) = events.last() else {+ return Err("finish should emit a terminal Responses event".into());+ };+ assert_eq!(terminal["type"], "response.incomplete");+ assert_eq!(terminal["response"]["status"], "incomplete");+ assert_eq!(+ terminal["response"]["incomplete_details"],+ json!({"reason": "max_output_tokens"})+ );+ assert_eq!(terminal["response"]["output"][0]["status"], "incomplete");+ Ok(())+}++// Verifies an Anthropic-vocabulary token-limit stop also terminates with response.incomplete.+#[test]+fn anthropic_max_tokens_stop_translates_to_responses_incomplete_event() -> TestResult {+ let engine = TranslationEngine::default();+ let mut state =+ StreamTranslationState::new(WireFormat::AnthropicMessages, WireFormat::OpenAiResponses);+ let delta = json!({+ "type": "message_delta",+ "delta": {"stop_reason": "max_tokens"},+ "usage": {"output_tokens": 1}+ });++ let mut events = engine.translate_event(+ &mut state,+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiResponses,+ &delta,+ )?;+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);++ let Some(terminal) = events.last() else {+ return Err("finish should emit a terminal Responses event".into());+ };+ assert_eq!(terminal["type"], "response.incomplete");+ assert_eq!(+ terminal["response"]["incomplete_details"],+ json!({"reason": "max_output_tokens"})+ );+ Ok(())+}++// Verifies a streamed response.incomplete from a Responses upstream reaches a Chat client.+#[test]+fn responses_incomplete_event_translates_to_chat_length_finish() -> TestResult {+ let engine = TranslationEngine::default();+ let mut state =+ StreamTranslationState::new(WireFormat::OpenAiResponses, WireFormat::OpenAiChat);+ let incomplete = json!({+ "type": "response.incomplete",+ "response": {"status": "incomplete", "incomplete_details": {"reason": "max_output_tokens"}}+ });++ let mut events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &incomplete,+ )?;+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiChat)?);++ let Some(terminal) = events.last() else {+ return Err("finish should emit a terminal Chat chunk".into());+ };+ assert_eq!(terminal["choices"][0]["finish_reason"], "length");+ Ok(())+}