Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): attach responses reasoning items to their assistant turn instead of emitting empty chat messages (#99)
eric-liu-nvidia5ae2a0fe00bf
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex e92ec45b9..18e9125a7 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -277,6 +277,7 @@ fn decode_responses_input(let mut pending_tool_calls = Vec::new();let mut pending_tool_outputs = Vec::new();let mut deferred_messages = Vec::new();+ let mut pending_reasoning: Vec<ContentBlock> = Vec::new();for (index, item) in items.iter().enumerate() {let Some(item) = item.as_object() else {push_lossy(@@ -288,35 +289,55 @@ fn decode_responses_input(};match item.get("type").and_then(Value::as_str) {Some("message") => {- let message = Message {- role: request_role_from_responses(- item.get("role").and_then(Value::as_str),- &format!("$.input[{index}].role"),- )?,- content: decode_responses_content(- item.get("content").unwrap_or(&Value::Null),- ),- };+ let role = request_role_from_responses(+ item.get("role").and_then(Value::as_str),+ &format!("$.input[{index}].role"),+ )?;+ let mut content =+ decode_responses_content(item.get("content").unwrap_or(&Value::Null));+ // Reasoning that precedes an assistant message belongs to+ // that turn; fold it in so it never surfaces as its own+ // (empty-looking) assistant message downstream.+ if role == Role::Assistant+ && pending_tool_calls.is_empty()+ && !pending_reasoning.is_empty()+ {+ let mut merged = std::mem::take(&mut pending_reasoning);+ merged.append(&mut content);+ content = merged;+ }+ if role != Role::Assistant {+ flush_unattached_responses_reasoning(+ &mut messages,+ &mut pending_tool_calls,+ &mut pending_tool_outputs,+ &mut deferred_messages,+ &mut pending_reasoning,+ );+ }push_responses_non_tool_message(&mut messages,&mut pending_tool_calls,&mut pending_tool_outputs,&mut deferred_messages,- message,+ &mut pending_reasoning,+ Message { role, content },);}Some("reasoning") => {- let message = Message {- role: Role::Assistant,- content: decode_responses_reasoning_item(item),- };- push_responses_non_tool_message(- &mut messages,- &mut pending_tool_calls,- &mut pending_tool_outputs,- &mut deferred_messages,- message,- );+ // A reasoning item after a completed tool block opens the+ // next assistant turn: flush the finished block so this+ // reasoning attaches to the turn that produced it.+ if !pending_tool_outputs.is_empty() {+ flush_responses_tool_block(+ &mut messages,+ &mut pending_tool_calls,+ &mut pending_tool_outputs,+ &mut deferred_messages,+ &mut pending_reasoning,+ );+ }+ pending_reasoning.extend(decode_responses_reasoning_item(item));}Some("function_call") => {if !pending_tool_outputs.is_empty() {@@ -325,6 +346,7 @@ fn decode_responses_input(&mut pending_tool_calls,&mut pending_tool_outputs,&mut deferred_messages,+ &mut pending_reasoning,);}let id = item@@ -369,11 +391,19 @@ fn decode_responses_input(raw: Value::Object(item.clone()),}],};+ flush_unattached_responses_reasoning(+ &mut messages,+ &mut pending_tool_calls,+ &mut pending_tool_outputs,+ &mut deferred_messages,+ &mut pending_reasoning,+ );push_responses_non_tool_message(&mut messages,&mut pending_tool_calls,&mut pending_tool_outputs,&mut deferred_messages,+ &mut pending_reasoning,message,);}@@ -385,8 +415,17 @@ fn decode_responses_input(&mut pending_tool_calls,&mut pending_tool_outputs,&mut deferred_messages,+ &mut pending_reasoning,);}+ // Trailing reasoning with no assistant turn to join keeps the+ // historical standalone-message shape.+ if !pending_reasoning.is_empty() {+ messages.push(Message {+ role: Role::Assistant,+ content: pending_reasoning,+ });+ }Ok(messages)}other => Ok(vec![Message::text(@@ -402,6 +441,7 @@ fn push_responses_non_tool_message(pending_tool_calls: &mut Vec<ToolCall>,pending_tool_outputs: &mut Vec<ToolResult>,deferred_messages: &mut Vec<Message>,+ pending_reasoning: &mut Vec<ContentBlock>,message: Message,) {if !pending_tool_calls.is_empty() && pending_tool_outputs.is_empty() {@@ -414,17 +454,48 @@ fn push_responses_non_tool_message(pending_tool_calls,pending_tool_outputs,deferred_messages,+ pending_reasoning,);}messages.push(message);}+// Emits reasoning that cannot join an assistant turn as a standalone assistant+// message at its original position. While tool calls are pending, the+// reasoning belongs to the in-flight turn and stays pending for the flush.+fn flush_unattached_responses_reasoning(+ messages: &mut Vec<Message>,+ pending_tool_calls: &mut Vec<ToolCall>,+ pending_tool_outputs: &mut Vec<ToolResult>,+ deferred_messages: &mut Vec<Message>,+ pending_reasoning: &mut Vec<ContentBlock>,+) {+ if pending_reasoning.is_empty() || !pending_tool_calls.is_empty() {+ return;+ }+ let message = Message {+ role: Role::Assistant,+ content: std::mem::take(pending_reasoning),+ };+ push_responses_non_tool_message(+ messages,+ pending_tool_calls,+ pending_tool_outputs,+ deferred_messages,+ pending_reasoning,+ message,+ );+}+// Flushes pending function calls and matching outputs while preserving adjacency.+// Pending reasoning rides in the same assistant message as the tool calls so a+// Codex reasoning + function_call turn stays one assistant turn downstream.fn flush_responses_tool_block(messages: &mut Vec<Message>,pending_tool_calls: &mut Vec<ToolCall>,pending_tool_outputs: &mut Vec<ToolResult>,deferred_messages: &mut Vec<Message>,+ pending_reasoning: &mut Vec<ContentBlock>,) {let tool_call_ids = pending_tool_calls.iter()@@ -432,12 +503,15 @@ fn flush_responses_tool_block(.collect::<HashSet<_>>();if !pending_tool_calls.is_empty() {+ let mut content = std::mem::take(pending_reasoning);+ content.extend(+ std::mem::take(pending_tool_calls)+ .into_iter()+ .map(ContentBlock::ToolCall),+ );messages.push(Message {role: Role::Assistant,- content: std::mem::take(pending_tool_calls)- .into_iter()- .map(ContentBlock::ToolCall)- .collect(),+ content,});}diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex ea15c63db..ee24591d8 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -742,6 +742,155 @@ fn responses_chat_compatible_extensions_survive_to_openai_chat() -> TestResult {Ok(())}+// Verifies Codex-style reasoning items attach to the turn's tool-call message+// instead of surfacing as fabricated empty assistant chat messages.+#[test]+fn responses_reasoning_items_attach_to_tool_call_turn_for_openai_chat() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "big-reasoner",+ "input": [+ {"type": "message", "role": "user", "content": "Fix pip"},+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "\n\n"}]+ },+ {+ "type": "reasoning",+ "summary": [],+ "content": [{"type": "reasoning_text", "text": "Check the python setup."}]+ },+ {+ "type": "function_call",+ "name": "shell",+ "call_id": "call-1",+ "arguments": "{\"command\":\"pip3 --version\"}"+ },+ {"type": "function_call_output", "call_id": "call-1", "output": "no pip"}+ ]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ let messages = output["messages"]+ .as_array()+ .ok_or("messages is not an array")?;+ assert!(+ !messages+ .iter()+ .any(|message| message["role"] == "assistant" && message["content"] == ""),+ "reasoning item leaked an empty assistant message: {messages:?}"+ );+ assert_eq!(messages.len(), 4);+ assert_eq!(messages[1], json!({"role": "assistant", "content": "\n\n"}));+ assert_eq!(messages[2]["tool_calls"][0]["id"], "call-1");+ assert_eq!(messages[3]["role"], "tool");+ Ok(())+}++// Verifies a reasoning item merges into the assistant message that follows it.+#[test]+fn responses_reasoning_item_merges_into_next_assistant_message_for_openai_chat() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "gpt-5",+ "input": [+ {"type": "message", "role": "user", "content": "Check the file"},+ {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Reading."}]},+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "Let me check."}]+ }+ ]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(+ output["messages"],+ json!([+ {"role": "user", "content": "Check the file"},+ {"role": "assistant", "content": "Let me check."}+ ])+ );+ Ok(())+}++// Verifies merged reasoning re-emerges as a Responses reasoning item ahead of+// the turn's function call when encoding back to the Responses format.+#[test]+fn responses_reasoning_items_round_trip_through_decode_and_encode() -> TestResult {+ let engine = TranslationEngine::default();+ let policy = TranslationPolicy {+ preservation: switchyard_translation::PreservationPolicy::Disabled,+ ..TranslationPolicy::default()+ };+ let body = json!({+ "model": "gpt-5",+ "input": [+ {"type": "message", "role": "user", "content": "List files"},+ {+ "type": "reasoning",+ "summary": [],+ "content": [{"type": "reasoning_text", "text": "Simple ls."}]+ },+ {+ "type": "function_call",+ "name": "shell",+ "call_id": "call-ls",+ "arguments": "{\"command\":\"ls\"}"+ },+ {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"}+ ]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiResponses,+ &body,+ &policy,+ )?+ .body;++ let input = output["input"].as_array().ok_or("input is not an array")?;+ let item_types = input+ .iter()+ .map(|item| item["type"].as_str().unwrap_or_default())+ .collect::<Vec<_>>();+ assert_eq!(+ item_types,+ vec![+ "message",+ "reasoning",+ "function_call",+ "function_call_output"+ ]+ );+ assert_eq!(+ input[1]["content"],+ json!([{"type": "reasoning_text", "text": "Simple ls."}])+ );+ assert_eq!(input[2]["call_id"], "call-ls");+ Ok(())+}+// Verifies Responses JSON schema text format maps to Chat response_format shape.#[test]fn responses_json_schema_text_format_maps_to_chat_response_format() -> TestResult {diff --git a/tests/test_responses_openai_translation.py b/tests/test_responses_openai_translation.pyindex b3ff5ea74..dff117c0e 100644--- a/tests/test_responses_openai_translation.py+++ b/tests/test_responses_openai_translation.py@@ -655,6 +655,191 @@ class TestConvertResponsesRequestToChatCompletions:assert messages[0]["tool_calls"][0]["function"]["arguments"] == "[[...]]"assert messages[1]["content"] == "[[...]]"+ def test_codex_reasoning_items_do_not_become_empty_assistant_messages(self):+ """Codex replays each model turn as message + reasoning + function_call items.++ The reasoning item must ride with the turn's tool-call message instead of+ surfacing as a fabricated ``{"role": "assistant", "content": ""}`` chat+ message; those empty turns accumulate every round-trip and degrade the+ upstream model until it stops emitting tool calls.+ """+ body = {+ "model": "big-reasoner",+ "instructions": "You are a coding agent.",+ "input": [+ {+ "type": "message",+ "role": "user",+ "content": [{"type": "input_text", "text": "Fix the broken pip install."}],+ },+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "\n\n"}],+ },+ {+ "type": "reasoning",+ "summary": [],+ "content": [+ {"type": "reasoning_text", "text": "Let me check the python setup."}+ ],+ "encrypted_content": None,+ },+ {+ "type": "function_call",+ "name": "shell_command",+ "arguments": '{"command":"pip3 --version","workdir":"/app"}',+ "call_id": "call-1",+ },+ {+ "type": "function_call_output",+ "call_id": "call-1",+ "output": "ModuleNotFoundError: No module named 'pip'",+ },+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "\n\n"}],+ },+ {+ "type": "reasoning",+ "summary": [],+ "content": [{"type": "reasoning_text", "text": "pip is missing; ensurepip."}],+ "encrypted_content": None,+ },+ {+ "type": "function_call",+ "name": "shell_command",+ "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}',+ "call_id": "call-2",+ },+ {+ "type": "function_call_output",+ "call_id": "call-2",+ "output": "Successfully installed pip",+ },+ ],+ }+ result = _responses_request_to_chat(body)++ assert result["messages"] == [+ {"role": "system", "content": "You are a coding agent."},+ {"role": "user", "content": "Fix the broken pip install."},+ {"role": "assistant", "content": "\n\n"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call-1",+ "type": "function",+ "function": {+ "name": "shell_command",+ "arguments": '{"command":"pip3 --version","workdir":"/app"}',+ },+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call-1",+ "content": "ModuleNotFoundError: No module named 'pip'",+ },+ {"role": "assistant", "content": "\n\n"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call-2",+ "type": "function",+ "function": {+ "name": "shell_command",+ "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}',+ },+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call-2",+ "content": "Successfully installed pip",+ },+ ]++ def test_reasoning_item_merges_into_following_assistant_message(self):+ """Native item ordering puts reasoning before the assistant message it belongs to."""+ body = {+ "model": "gpt-5",+ "input": [+ {"type": "message", "role": "user", "content": "Check the file"},+ {+ "type": "reasoning",+ "summary": [{"type": "summary_text", "text": "Reading it now."}],+ },+ {+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "Let me check."}],+ },+ {+ "type": "function_call",+ "name": "read_file",+ "call_id": "call-r",+ "arguments": '{"path":"a.py"}',+ },+ {"type": "function_call_output", "call_id": "call-r", "output": "print(1)"},+ ],+ }+ result = _responses_request_to_chat(body)++ assert result["messages"] == [+ {"role": "user", "content": "Check the file"},+ {"role": "assistant", "content": "Let me check."},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call-r",+ "type": "function",+ "function": {"name": "read_file", "arguments": '{"path":"a.py"}'},+ }+ ],+ },+ {"role": "tool", "tool_call_id": "call-r", "content": "print(1)"},+ ]++ def test_reasoning_only_turn_does_not_add_empty_assistant_message(self):+ """A reasoning item directly before a tool call must not split the turn."""+ body = {+ "model": "gpt-5",+ "input": [+ {"type": "message", "role": "user", "content": "List files"},+ {+ "type": "reasoning",+ "summary": [],+ "content": [{"type": "reasoning_text", "text": "Simple ls."}],+ },+ {+ "type": "function_call",+ "name": "shell",+ "call_id": "call-ls",+ "arguments": '{"command":"ls"}',+ },+ {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"},+ ],+ }+ result = _responses_request_to_chat(body)++ empty_assistants = [+ message+ for message in result["messages"]+ if message.get("role") == "assistant" and message.get("content") == ""+ ]+ assert empty_assistants == []+ assert result["messages"][1]["tool_calls"][0]["id"] == "call-ls"+# ---------------------------------------------------------------------------# Response conversion tests@@ -820,3 +1005,133 @@ class TestConvertChatResponseToResponses:assert result["output"][0]["content"][0]["text"] == "Let me check that."assert result["output"][1]["type"] == "function_call"assert result["output"][1]["name"] == "lookup"++ def test_reasoning_content_with_tool_calls_has_no_empty_message_item(self):+ """Reasoning + tool calls with empty text must not fabricate an empty message item."""+ response = {+ "id": "chatcmpl-reason",+ "model": "big-reasoner",+ "choices": [+ {+ "message": {+ "role": "assistant",+ "content": None,+ "reasoning_content": "Inspect the repository first.",+ "tool_calls": [+ {+ "id": "call-9",+ "type": "function",+ "function": {+ "name": "shell",+ "arguments": '{"command":"ls"}',+ },+ }+ ],+ },+ "finish_reason": "tool_calls",+ }+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},+ }+ result = _chat_response_to_responses(response)++ assert [item["type"] for item in result["output"]] == ["reasoning", "function_call"]+ reasoning = result["output"][0]+ assert reasoning["content"] == [+ {"type": "reasoning_text", "text": "Inspect the repository first."}+ ]+ function_call = result["output"][1]+ assert function_call["call_id"] == "call-9"+ assert function_call["name"] == "shell"+ assert function_call["arguments"] == '{"command": "ls"}'+++# ---------------------------------------------------------------------------+# Streaming conversion tests (chat chunks -> Responses SSE events)+# ---------------------------------------------------------------------------+++def _chat_chunk(delta: dict, finish: str | None = None) -> dict:+ return {+ "id": "chatcmpl-stream",+ "object": "chat.completion.chunk",+ "model": "big-reasoner",+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],+ }+++async def _translate_chat_stream_to_responses(chunks: list[dict]) -> list[dict]:+ async def _source():+ for chunk in chunks:+ yield chunk++ return [+ event+ async for event in ENGINE.translate_stream("openai_chat", "openai_responses", _source())+ ]+++class TestChatStreamToResponsesSse:+ """Pins the SSE contract codex consumes for streamed tool calls."""++ async def test_tool_call_stream_produces_well_formed_function_call_events(self):+ events = await _translate_chat_stream_to_responses(+ [+ _chat_chunk({"role": "assistant", "content": ""}),+ _chat_chunk({"reasoning_content": "Need to inspect the repo."}),+ _chat_chunk({"content": "\n\n"}),+ _chat_chunk(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call-9",+ "type": "function",+ "function": {"name": "shell", "arguments": '{"command":'},+ }+ ]+ }+ ),+ _chat_chunk(+ {"tool_calls": [{"index": 0, "function": {"arguments": '"ls"}'}}]}+ ),+ _chat_chunk({}, finish="tool_calls"),+ ]+ )++ added = [+ event["item"]+ for event in events+ if event["type"] == "response.output_item.added"+ and event["item"]["type"] == "function_call"+ ]+ assert len(added) == 1+ assert added[0]["call_id"] == "call-9"+ assert added[0]["name"] == "shell"++ argument_deltas = [+ event["delta"]+ for event in events+ if event["type"] == "response.function_call_arguments.delta"+ ]+ assert "".join(argument_deltas) == '{"command":"ls"}'++ done = [+ event["item"]+ for event in events+ if event["type"] == "response.output_item.done"+ and event["item"]["type"] == "function_call"+ ]+ assert len(done) == 1+ assert done[0]["call_id"] == "call-9"+ assert done[0]["name"] == "shell"+ assert done[0]["arguments"] == '{"command":"ls"}'+ assert done[0]["status"] == "completed"++ completed = [event for event in events if event["type"] == "response.completed"]+ assert len(completed) == 1+ output_types = [item["type"] for item in completed[0]["response"]["output"]]+ assert output_types == ["reasoning", "message", "function_call"]+ function_item = completed[0]["response"]["output"][2]+ assert function_item["call_id"] == "call-9"+ assert function_item["arguments"] == '{"command":"ls"}'