Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): classify Responses instruction roles inside decoder state machine (#523)
Atharva-Kanherkar01acc792e6fb
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex b3a7f7f2c..03e11f15b 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -18,7 +18,7 @@ use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};use crate::llm::{- AggLlmResponse, ContentBlock, LlmRequest, MediaSource, Message, OutputParams,+ AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, MediaSource, Message, OutputParams,ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,};@@ -82,11 +82,13 @@ impl FormatCodec for OpenAiResponsesCodec {}],});}- request.messages = decode_responses_input(+ let (messages, instructions) = decode_responses_input(body.get("input").unwrap_or(&Value::String(String::new())),&mut diagnostics,policy,)?;+ request.messages = messages;+ request.instructions.extend(instructions);let mut tool_namespaces = Map::new();request.tools = decode_responses_tools(body.get("tools"), &mut tool_namespaces);request.tool_choice = body@@ -322,16 +324,23 @@ impl FormatCodec for OpenAiResponsesCodec {}}-// Decodes Responses `input` into ordered normalized messages.+/// Decodes Responses `input` into ordered normalized messages and inline+/// instruction blocks.+///+/// Inline `system` and `developer` message items are returned as instruction+/// blocks rather than conversation turns. Classifying them here, before any+/// reasoning or tool-call state-machine transitions, prevents an instruction+/// item from flushing pending reasoning or disturbing tool-call grouping.fn decode_responses_input(value: &Value,diagnostics: &mut Vec<TranslationDiagnostic>,policy: &TranslationPolicy,-) -> Result<Vec<Message>> {+) -> Result<(Vec<Message>, Vec<InstructionBlock>)> {match value {- Value::String(text) => Ok(vec![Message::text(Role::User, text)]),+ Value::String(text) => Ok((vec![Message::text(Role::User, text)], Vec::new())),Value::Array(items) => {let mut messages = Vec::new();+ let mut instructions = Vec::new();let mut pending_tool_calls = Vec::new();let mut pending_tool_outputs = Vec::new();let mut deferred_messages = Vec::new();@@ -365,8 +374,17 @@ fn decode_responses_input(item.get("role").and_then(Value::as_str),&format!("$.input[{index}].role"),)?;- let mut content =+ let content =decode_responses_content(item.get("content").unwrap_or(&Value::Null));+ // Inline system and developer input items are instructions, not+ // conversation turns. Classify them before any state-machine+ // transitions so they do not flush pending reasoning or disturb+ // tool-call grouping.+ if matches!(role, Role::System | Role::Developer) {+ instructions.push(InstructionBlock { role, content });+ continue;+ }+ let mut content = content;// Reasoning that precedes an assistant message belongs to// that turn; fold it in so it never surfaces as its own// (empty-looking) assistant message downstream.@@ -519,7 +537,7 @@ fn decode_responses_input(content: pending_reasoning,});}- Ok(messages)+ Ok((messages, instructions))}_ => Err(TranslationError::InvalidType {path: "$.input".to_string(),@@ -528,7 +546,7 @@ fn decode_responses_input(}}-// Places non-tool input items without breaking Responses tool-call adjacency.+/// Places non-tool input items without breaking Responses tool-call adjacency.fn push_responses_non_tool_message(messages: &mut Vec<Message>,pending_tool_calls: &mut Vec<ToolCall>,@@ -553,9 +571,9 @@ fn push_responses_non_tool_message(messages.push(message);}-// Emits reasoning that cannot join an assistant turn as a standalone assistant-// message at its original position. While tool calls are pending, the-// reasoning belongs to the in-flight turn and stays pending for the flush.+/// Emits reasoning that cannot join an assistant turn as a standalone assistant+/// message at its original position. While tool calls are pending, the+/// reasoning belongs to the in-flight turn and stays pending for the flush.fn flush_unattached_responses_reasoning(messages: &mut Vec<Message>,pending_tool_calls: &mut Vec<ToolCall>,@@ -580,9 +598,9 @@ fn flush_unattached_responses_reasoning();}-// Flushes pending function calls and matching outputs while preserving adjacency.-// Pending reasoning rides in the same assistant message as the tool calls so a-// Codex reasoning + function_call turn stays one assistant turn downstream.+/// Flushes pending function calls and matching outputs while preserving adjacency.+/// Pending reasoning rides in the same assistant message as the tool calls so a+/// Codex reasoning + function_call turn stays one assistant turn downstream.fn flush_responses_tool_block(messages: &mut Vec<Message>,pending_tool_calls: &mut Vec<ToolCall>,diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 56ae8d6ea..d17fa7f99 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -732,13 +732,18 @@ fn responses_unknown_input_item_is_preserved_for_openai_chat() -> TestResult {Ok(())}-// Responses accepts message-shaped input items without an explicit discriminator.+// Responses accepts message-shaped input items without an explicit discriminator, and inline+// system and developer items keep their roles instead of being demoted to user.#[test]-fn responses_input_message_without_type_translates_normally() -> TestResult {+fn responses_input_messages_translate_with_instruction_roles_intact() -> TestResult {let engine = TranslationEngine::default();let body = json!({"model": "gpt-4",- "input": [{"role": "user", "content": "hello"}]+ "input": [+ {"type": "message", "role": "system", "content": "Be terse."},+ {"type": "message", "role": "developer", "content": "Return JSON only."},+ {"role": "user", "content": "hello"}+ ]});let output = engine@@ -752,11 +757,46 @@ fn responses_input_message_without_type_translates_normally() -> TestResult {assert_eq!(output["messages"],- json!([{"role": "user", "content": "hello"}])+ json!([+ {"role": "system", "content": "Be terse."},+ {"role": "developer", "content": "Return JSON only."},+ {"role": "user", "content": "hello"}+ ]));Ok(())}+// Inline instruction items must not detach pending reasoning from the assistant+// turn that produced it.+#[test]+fn responses_inline_instruction_does_not_detach_reasoning() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "gpt-4",+ "input": [+ {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "thinking..."}], "summary": []},+ {"type": "message", "role": "system", "content": "Be terse."},+ {"type": "message", "role": "assistant", "content": "hello"}+ ]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(output["messages"].as_array().map(Vec::len), Some(2));+ assert_eq!(output["messages"][0]["role"], "system");+ assert_eq!(output["messages"][1]["role"], "assistant");+ assert_eq!(output["messages"][1]["content"], "hello");+ assert_eq!(output["messages"][1]["reasoning"], "thinking...");+ Ok(())+}+// A discriminator-less object that is not message-shaped must not silently become prompt text.#[test]fn responses_input_without_type_or_message_shape_is_rejected() {