Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): attach responses reasoning items to their assistant turn instead of emitting empty chat messages (#99)

eric-liu-nvidia5ae2a0fe00bf
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index e92ec45b9..18e9125a7 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -277,6 +277,7 @@ fn decode_responses_input(
let mut pending_tool_calls = Vec::new();
let mut pending_tool_outputs = Vec::new();
let mut deferred_messages = Vec::new();
+ let mut pending_reasoning: Vec<ContentBlock> = Vec::new();
for (index, item) in items.iter().enumerate() {
let Some(item) = item.as_object() else {
push_lossy(
@@ -288,35 +289,55 @@ fn decode_responses_input(
};
match item.get("type").and_then(Value::as_str) {
Some("message") => {
- let message = Message {
- role: request_role_from_responses(
- item.get("role").and_then(Value::as_str),
- &format!("$.input[{index}].role"),
- )?,
- content: decode_responses_content(
- item.get("content").unwrap_or(&Value::Null),
- ),
- };
+ let role = request_role_from_responses(
+ item.get("role").and_then(Value::as_str),
+ &format!("$.input[{index}].role"),
+ )?;
+ let mut content =
+ decode_responses_content(item.get("content").unwrap_or(&Value::Null));
+ // Reasoning that precedes an assistant message belongs to
+ // that turn; fold it in so it never surfaces as its own
+ // (empty-looking) assistant message downstream.
+ if role == Role::Assistant
+ && pending_tool_calls.is_empty()
+ && !pending_reasoning.is_empty()
+ {
+ let mut merged = std::mem::take(&mut pending_reasoning);
+ merged.append(&mut content);
+ content = merged;
+ }
+ if role != Role::Assistant {
+ flush_unattached_responses_reasoning(
+ &mut messages,
+ &mut pending_tool_calls,
+ &mut pending_tool_outputs,
+ &mut deferred_messages,
+ &mut pending_reasoning,
+ );
+ }
push_responses_non_tool_message(
&mut messages,
&mut pending_tool_calls,
&mut pending_tool_outputs,
&mut deferred_messages,
- message,
+ &mut pending_reasoning,
+ Message { role, content },
);
}
Some("reasoning") => {
- let message = Message {
- role: Role::Assistant,
- content: decode_responses_reasoning_item(item),
- };
- push_responses_non_tool_message(
- &mut messages,
- &mut pending_tool_calls,
- &mut pending_tool_outputs,
- &mut deferred_messages,
- message,
- );
+ // A reasoning item after a completed tool block opens the
+ // next assistant turn: flush the finished block so this
+ // reasoning attaches to the turn that produced it.
+ if !pending_tool_outputs.is_empty() {
+ flush_responses_tool_block(
+ &mut messages,
+ &mut pending_tool_calls,
+ &mut pending_tool_outputs,
+ &mut deferred_messages,
+ &mut pending_reasoning,
+ );
+ }
+ pending_reasoning.extend(decode_responses_reasoning_item(item));
}
Some("function_call") => {
if !pending_tool_outputs.is_empty() {
@@ -325,6 +346,7 @@ fn decode_responses_input(
&mut pending_tool_calls,
&mut pending_tool_outputs,
&mut deferred_messages,
+ &mut pending_reasoning,
);
}
let id = item
@@ -369,11 +391,19 @@ fn decode_responses_input(
raw: Value::Object(item.clone()),
}],
};
+ flush_unattached_responses_reasoning(
+ &mut messages,
+ &mut pending_tool_calls,
+ &mut pending_tool_outputs,
+ &mut deferred_messages,
+ &mut pending_reasoning,
+ );
push_responses_non_tool_message(
&mut messages,
&mut pending_tool_calls,
&mut pending_tool_outputs,
&mut deferred_messages,
+ &mut pending_reasoning,
message,
);
}
@@ -385,8 +415,17 @@ fn decode_responses_input(
&mut pending_tool_calls,
&mut pending_tool_outputs,
&mut deferred_messages,
+ &mut pending_reasoning,
);
}
+ // Trailing reasoning with no assistant turn to join keeps the
+ // historical standalone-message shape.
+ if !pending_reasoning.is_empty() {
+ messages.push(Message {
+ role: Role::Assistant,
+ content: pending_reasoning,
+ });
+ }
Ok(messages)
}
other => Ok(vec![Message::text(
@@ -402,6 +441,7 @@ fn push_responses_non_tool_message(
pending_tool_calls: &mut Vec<ToolCall>,
pending_tool_outputs: &mut Vec<ToolResult>,
deferred_messages: &mut Vec<Message>,
+ pending_reasoning: &mut Vec<ContentBlock>,
message: Message,
) {
if !pending_tool_calls.is_empty() && pending_tool_outputs.is_empty() {
@@ -414,17 +454,48 @@ fn push_responses_non_tool_message(
pending_tool_calls,
pending_tool_outputs,
deferred_messages,
+ pending_reasoning,
);
}
messages.push(message);
}
+// Emits reasoning that cannot join an assistant turn as a standalone assistant
+// message at its original position. While tool calls are pending, the
+// reasoning belongs to the in-flight turn and stays pending for the flush.
+fn flush_unattached_responses_reasoning(
+ messages: &mut Vec<Message>,
+ pending_tool_calls: &mut Vec<ToolCall>,
+ pending_tool_outputs: &mut Vec<ToolResult>,
+ deferred_messages: &mut Vec<Message>,
+ pending_reasoning: &mut Vec<ContentBlock>,
+) {
+ if pending_reasoning.is_empty() || !pending_tool_calls.is_empty() {
+ return;
+ }
+ let message = Message {
+ role: Role::Assistant,
+ content: std::mem::take(pending_reasoning),
+ };
+ push_responses_non_tool_message(
+ messages,
+ pending_tool_calls,
+ pending_tool_outputs,
+ deferred_messages,
+ pending_reasoning,
+ message,
+ );
+}
+
// Flushes pending function calls and matching outputs while preserving adjacency.
+// Pending reasoning rides in the same assistant message as the tool calls so a
+// Codex reasoning + function_call turn stays one assistant turn downstream.
fn flush_responses_tool_block(
messages: &mut Vec<Message>,
pending_tool_calls: &mut Vec<ToolCall>,
pending_tool_outputs: &mut Vec<ToolResult>,
deferred_messages: &mut Vec<Message>,
+ pending_reasoning: &mut Vec<ContentBlock>,
) {
let tool_call_ids = pending_tool_calls
.iter()
@@ -432,12 +503,15 @@ fn flush_responses_tool_block(
.collect::<HashSet<_>>();
if !pending_tool_calls.is_empty() {
+ let mut content = std::mem::take(pending_reasoning);
+ content.extend(
+ std::mem::take(pending_tool_calls)
+ .into_iter()
+ .map(ContentBlock::ToolCall),
+ );
messages.push(Message {
role: Role::Assistant,
- content: std::mem::take(pending_tool_calls)
- .into_iter()
- .map(ContentBlock::ToolCall)
- .collect(),
+ content,
});
}
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index ea15c63db..ee24591d8 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -742,6 +742,155 @@ fn responses_chat_compatible_extensions_survive_to_openai_chat() -> TestResult {
Ok(())
}
+// Verifies Codex-style reasoning items attach to the turn's tool-call message
+// instead of surfacing as fabricated empty assistant chat messages.
+#[test]
+fn responses_reasoning_items_attach_to_tool_call_turn_for_openai_chat() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "big-reasoner",
+ "input": [
+ {"type": "message", "role": "user", "content": "Fix pip"},
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "\n\n"}]
+ },
+ {
+ "type": "reasoning",
+ "summary": [],
+ "content": [{"type": "reasoning_text", "text": "Check the python setup."}]
+ },
+ {
+ "type": "function_call",
+ "name": "shell",
+ "call_id": "call-1",
+ "arguments": "{\"command\":\"pip3 --version\"}"
+ },
+ {"type": "function_call_output", "call_id": "call-1", "output": "no pip"}
+ ]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ let messages = output["messages"]
+ .as_array()
+ .ok_or("messages is not an array")?;
+ assert!(
+ !messages
+ .iter()
+ .any(|message| message["role"] == "assistant" && message["content"] == ""),
+ "reasoning item leaked an empty assistant message: {messages:?}"
+ );
+ assert_eq!(messages.len(), 4);
+ assert_eq!(messages[1], json!({"role": "assistant", "content": "\n\n"}));
+ assert_eq!(messages[2]["tool_calls"][0]["id"], "call-1");
+ assert_eq!(messages[3]["role"], "tool");
+ Ok(())
+}
+
+// Verifies a reasoning item merges into the assistant message that follows it.
+#[test]
+fn responses_reasoning_item_merges_into_next_assistant_message_for_openai_chat() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-5",
+ "input": [
+ {"type": "message", "role": "user", "content": "Check the file"},
+ {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Reading."}]},
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "Let me check."}]
+ }
+ ]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(
+ output["messages"],
+ json!([
+ {"role": "user", "content": "Check the file"},
+ {"role": "assistant", "content": "Let me check."}
+ ])
+ );
+ Ok(())
+}
+
+// Verifies merged reasoning re-emerges as a Responses reasoning item ahead of
+// the turn's function call when encoding back to the Responses format.
+#[test]
+fn responses_reasoning_items_round_trip_through_decode_and_encode() -> TestResult {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy {
+ preservation: switchyard_translation::PreservationPolicy::Disabled,
+ ..TranslationPolicy::default()
+ };
+ let body = json!({
+ "model": "gpt-5",
+ "input": [
+ {"type": "message", "role": "user", "content": "List files"},
+ {
+ "type": "reasoning",
+ "summary": [],
+ "content": [{"type": "reasoning_text", "text": "Simple ls."}]
+ },
+ {
+ "type": "function_call",
+ "name": "shell",
+ "call_id": "call-ls",
+ "arguments": "{\"command\":\"ls\"}"
+ },
+ {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"}
+ ]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiResponses,
+ &body,
+ &policy,
+ )?
+ .body;
+
+ let input = output["input"].as_array().ok_or("input is not an array")?;
+ let item_types = input
+ .iter()
+ .map(|item| item["type"].as_str().unwrap_or_default())
+ .collect::<Vec<_>>();
+ assert_eq!(
+ item_types,
+ vec![
+ "message",
+ "reasoning",
+ "function_call",
+ "function_call_output"
+ ]
+ );
+ assert_eq!(
+ input[1]["content"],
+ json!([{"type": "reasoning_text", "text": "Simple ls."}])
+ );
+ assert_eq!(input[2]["call_id"], "call-ls");
+ Ok(())
+}
+
// Verifies Responses JSON schema text format maps to Chat response_format shape.
#[test]
fn responses_json_schema_text_format_maps_to_chat_response_format() -> TestResult {
diff --git a/tests/test_responses_openai_translation.py b/tests/test_responses_openai_translation.py
index b3ff5ea74..dff117c0e 100644
--- a/tests/test_responses_openai_translation.py
+++ b/tests/test_responses_openai_translation.py
@@ -655,6 +655,191 @@ class TestConvertResponsesRequestToChatCompletions:
assert messages[0]["tool_calls"][0]["function"]["arguments"] == "[[...]]"
assert messages[1]["content"] == "[[...]]"
+ def test_codex_reasoning_items_do_not_become_empty_assistant_messages(self):
+ """Codex replays each model turn as message + reasoning + function_call items.
+
+ The reasoning item must ride with the turn's tool-call message instead of
+ surfacing as a fabricated ``{"role": "assistant", "content": ""}`` chat
+ message; those empty turns accumulate every round-trip and degrade the
+ upstream model until it stops emitting tool calls.
+ """
+ body = {
+ "model": "big-reasoner",
+ "instructions": "You are a coding agent.",
+ "input": [
+ {
+ "type": "message",
+ "role": "user",
+ "content": [{"type": "input_text", "text": "Fix the broken pip install."}],
+ },
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "\n\n"}],
+ },
+ {
+ "type": "reasoning",
+ "summary": [],
+ "content": [
+ {"type": "reasoning_text", "text": "Let me check the python setup."}
+ ],
+ "encrypted_content": None,
+ },
+ {
+ "type": "function_call",
+ "name": "shell_command",
+ "arguments": '{"command":"pip3 --version","workdir":"/app"}',
+ "call_id": "call-1",
+ },
+ {
+ "type": "function_call_output",
+ "call_id": "call-1",
+ "output": "ModuleNotFoundError: No module named 'pip'",
+ },
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "\n\n"}],
+ },
+ {
+ "type": "reasoning",
+ "summary": [],
+ "content": [{"type": "reasoning_text", "text": "pip is missing; ensurepip."}],
+ "encrypted_content": None,
+ },
+ {
+ "type": "function_call",
+ "name": "shell_command",
+ "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}',
+ "call_id": "call-2",
+ },
+ {
+ "type": "function_call_output",
+ "call_id": "call-2",
+ "output": "Successfully installed pip",
+ },
+ ],
+ }
+ result = _responses_request_to_chat(body)
+
+ assert result["messages"] == [
+ {"role": "system", "content": "You are a coding agent."},
+ {"role": "user", "content": "Fix the broken pip install."},
+ {"role": "assistant", "content": "\n\n"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call-1",
+ "type": "function",
+ "function": {
+ "name": "shell_command",
+ "arguments": '{"command":"pip3 --version","workdir":"/app"}',
+ },
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call-1",
+ "content": "ModuleNotFoundError: No module named 'pip'",
+ },
+ {"role": "assistant", "content": "\n\n"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call-2",
+ "type": "function",
+ "function": {
+ "name": "shell_command",
+ "arguments": '{"command":"python3 -m ensurepip","workdir":"/app"}',
+ },
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call-2",
+ "content": "Successfully installed pip",
+ },
+ ]
+
+ def test_reasoning_item_merges_into_following_assistant_message(self):
+ """Native item ordering puts reasoning before the assistant message it belongs to."""
+ body = {
+ "model": "gpt-5",
+ "input": [
+ {"type": "message", "role": "user", "content": "Check the file"},
+ {
+ "type": "reasoning",
+ "summary": [{"type": "summary_text", "text": "Reading it now."}],
+ },
+ {
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "Let me check."}],
+ },
+ {
+ "type": "function_call",
+ "name": "read_file",
+ "call_id": "call-r",
+ "arguments": '{"path":"a.py"}',
+ },
+ {"type": "function_call_output", "call_id": "call-r", "output": "print(1)"},
+ ],
+ }
+ result = _responses_request_to_chat(body)
+
+ assert result["messages"] == [
+ {"role": "user", "content": "Check the file"},
+ {"role": "assistant", "content": "Let me check."},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call-r",
+ "type": "function",
+ "function": {"name": "read_file", "arguments": '{"path":"a.py"}'},
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "call-r", "content": "print(1)"},
+ ]
+
+ def test_reasoning_only_turn_does_not_add_empty_assistant_message(self):
+ """A reasoning item directly before a tool call must not split the turn."""
+ body = {
+ "model": "gpt-5",
+ "input": [
+ {"type": "message", "role": "user", "content": "List files"},
+ {
+ "type": "reasoning",
+ "summary": [],
+ "content": [{"type": "reasoning_text", "text": "Simple ls."}],
+ },
+ {
+ "type": "function_call",
+ "name": "shell",
+ "call_id": "call-ls",
+ "arguments": '{"command":"ls"}',
+ },
+ {"type": "function_call_output", "call_id": "call-ls", "output": "a.py"},
+ ],
+ }
+ result = _responses_request_to_chat(body)
+
+ empty_assistants = [
+ message
+ for message in result["messages"]
+ if message.get("role") == "assistant" and message.get("content") == ""
+ ]
+ assert empty_assistants == []
+ assert result["messages"][1]["tool_calls"][0]["id"] == "call-ls"
+
# ---------------------------------------------------------------------------
# Response conversion tests
@@ -820,3 +1005,133 @@ class TestConvertChatResponseToResponses:
assert result["output"][0]["content"][0]["text"] == "Let me check that."
assert result["output"][1]["type"] == "function_call"
assert result["output"][1]["name"] == "lookup"
+
+ def test_reasoning_content_with_tool_calls_has_no_empty_message_item(self):
+ """Reasoning + tool calls with empty text must not fabricate an empty message item."""
+ response = {
+ "id": "chatcmpl-reason",
+ "model": "big-reasoner",
+ "choices": [
+ {
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "reasoning_content": "Inspect the repository first.",
+ "tool_calls": [
+ {
+ "id": "call-9",
+ "type": "function",
+ "function": {
+ "name": "shell",
+ "arguments": '{"command":"ls"}',
+ },
+ }
+ ],
+ },
+ "finish_reason": "tool_calls",
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
+ }
+ result = _chat_response_to_responses(response)
+
+ assert [item["type"] for item in result["output"]] == ["reasoning", "function_call"]
+ reasoning = result["output"][0]
+ assert reasoning["content"] == [
+ {"type": "reasoning_text", "text": "Inspect the repository first."}
+ ]
+ function_call = result["output"][1]
+ assert function_call["call_id"] == "call-9"
+ assert function_call["name"] == "shell"
+ assert function_call["arguments"] == '{"command": "ls"}'
+
+
+# ---------------------------------------------------------------------------
+# Streaming conversion tests (chat chunks -> Responses SSE events)
+# ---------------------------------------------------------------------------
+
+
+def _chat_chunk(delta: dict, finish: str | None = None) -> dict:
+ return {
+ "id": "chatcmpl-stream",
+ "object": "chat.completion.chunk",
+ "model": "big-reasoner",
+ "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
+ }
+
+
+async def _translate_chat_stream_to_responses(chunks: list[dict]) -> list[dict]:
+ async def _source():
+ for chunk in chunks:
+ yield chunk
+
+ return [
+ event
+ async for event in ENGINE.translate_stream("openai_chat", "openai_responses", _source())
+ ]
+
+
+class TestChatStreamToResponsesSse:
+ """Pins the SSE contract codex consumes for streamed tool calls."""
+
+ async def test_tool_call_stream_produces_well_formed_function_call_events(self):
+ events = await _translate_chat_stream_to_responses(
+ [
+ _chat_chunk({"role": "assistant", "content": ""}),
+ _chat_chunk({"reasoning_content": "Need to inspect the repo."}),
+ _chat_chunk({"content": "\n\n"}),
+ _chat_chunk(
+ {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call-9",
+ "type": "function",
+ "function": {"name": "shell", "arguments": '{"command":'},
+ }
+ ]
+ }
+ ),
+ _chat_chunk(
+ {"tool_calls": [{"index": 0, "function": {"arguments": '"ls"}'}}]}
+ ),
+ _chat_chunk({}, finish="tool_calls"),
+ ]
+ )
+
+ added = [
+ event["item"]
+ for event in events
+ if event["type"] == "response.output_item.added"
+ and event["item"]["type"] == "function_call"
+ ]
+ assert len(added) == 1
+ assert added[0]["call_id"] == "call-9"
+ assert added[0]["name"] == "shell"
+
+ argument_deltas = [
+ event["delta"]
+ for event in events
+ if event["type"] == "response.function_call_arguments.delta"
+ ]
+ assert "".join(argument_deltas) == '{"command":"ls"}'
+
+ done = [
+ event["item"]
+ for event in events
+ if event["type"] == "response.output_item.done"
+ and event["item"]["type"] == "function_call"
+ ]
+ assert len(done) == 1
+ assert done[0]["call_id"] == "call-9"
+ assert done[0]["name"] == "shell"
+ assert done[0]["arguments"] == '{"command":"ls"}'
+ assert done[0]["status"] == "completed"
+
+ completed = [event for event in events if event["type"] == "response.completed"]
+ assert len(completed) == 1
+ output_types = [item["type"] for item in completed[0]["response"]["output"]]
+ assert output_types == ["reasoning", "message", "function_call"]
+ function_item = completed[0]["response"]["output"][2]
+ assert function_item["call_id"] == "call-9"
+ assert function_item["arguments"] == '{"command":"ls"}'