Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): classify Responses instruction roles inside decoder state machine (#523)

Atharva-Kanherkar01acc792e6fb
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index b3a7f7f2c..03e11f15b 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -18,7 +18,7 @@ use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::{
- AggLlmResponse, ContentBlock, LlmRequest, MediaSource, Message, OutputParams,
+ AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, MediaSource, Message, OutputParams,
ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,
ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
@@ -82,11 +82,13 @@ impl FormatCodec for OpenAiResponsesCodec {
}],
});
}
- request.messages = decode_responses_input(
+ let (messages, instructions) = decode_responses_input(
body.get("input").unwrap_or(&Value::String(String::new())),
&mut diagnostics,
policy,
)?;
+ request.messages = messages;
+ request.instructions.extend(instructions);
let mut tool_namespaces = Map::new();
request.tools = decode_responses_tools(body.get("tools"), &mut tool_namespaces);
request.tool_choice = body
@@ -322,16 +324,23 @@ impl FormatCodec for OpenAiResponsesCodec {
}
}
-// Decodes Responses `input` into ordered normalized messages.
+/// Decodes Responses `input` into ordered normalized messages and inline
+/// instruction blocks.
+///
+/// Inline `system` and `developer` message items are returned as instruction
+/// blocks rather than conversation turns. Classifying them here, before any
+/// reasoning or tool-call state-machine transitions, prevents an instruction
+/// item from flushing pending reasoning or disturbing tool-call grouping.
fn decode_responses_input(
value: &Value,
diagnostics: &mut Vec<TranslationDiagnostic>,
policy: &TranslationPolicy,
-) -> Result<Vec<Message>> {
+) -> Result<(Vec<Message>, Vec<InstructionBlock>)> {
match value {
- Value::String(text) => Ok(vec![Message::text(Role::User, text)]),
+ Value::String(text) => Ok((vec![Message::text(Role::User, text)], Vec::new())),
Value::Array(items) => {
let mut messages = Vec::new();
+ let mut instructions = Vec::new();
let mut pending_tool_calls = Vec::new();
let mut pending_tool_outputs = Vec::new();
let mut deferred_messages = Vec::new();
@@ -365,8 +374,17 @@ fn decode_responses_input(
item.get("role").and_then(Value::as_str),
&format!("$.input[{index}].role"),
)?;
- let mut content =
+ let content =
decode_responses_content(item.get("content").unwrap_or(&Value::Null));
+ // Inline system and developer input items are instructions, not
+ // conversation turns. Classify them before any state-machine
+ // transitions so they do not flush pending reasoning or disturb
+ // tool-call grouping.
+ if matches!(role, Role::System | Role::Developer) {
+ instructions.push(InstructionBlock { role, content });
+ continue;
+ }
+ let mut content = content;
// Reasoning that precedes an assistant message belongs to
// that turn; fold it in so it never surfaces as its own
// (empty-looking) assistant message downstream.
@@ -519,7 +537,7 @@ fn decode_responses_input(
content: pending_reasoning,
});
}
- Ok(messages)
+ Ok((messages, instructions))
}
_ => Err(TranslationError::InvalidType {
path: "$.input".to_string(),
@@ -528,7 +546,7 @@ fn decode_responses_input(
}
}
-// Places non-tool input items without breaking Responses tool-call adjacency.
+/// Places non-tool input items without breaking Responses tool-call adjacency.
fn push_responses_non_tool_message(
messages: &mut Vec<Message>,
pending_tool_calls: &mut Vec<ToolCall>,
@@ -553,9 +571,9 @@ fn push_responses_non_tool_message(
messages.push(message);
}
-// Emits reasoning that cannot join an assistant turn as a standalone assistant
-// message at its original position. While tool calls are pending, the
-// reasoning belongs to the in-flight turn and stays pending for the flush.
+/// Emits reasoning that cannot join an assistant turn as a standalone assistant
+/// message at its original position. While tool calls are pending, the
+/// reasoning belongs to the in-flight turn and stays pending for the flush.
fn flush_unattached_responses_reasoning(
messages: &mut Vec<Message>,
pending_tool_calls: &mut Vec<ToolCall>,
@@ -580,9 +598,9 @@ fn flush_unattached_responses_reasoning(
);
}
-// Flushes pending function calls and matching outputs while preserving adjacency.
-// Pending reasoning rides in the same assistant message as the tool calls so a
-// Codex reasoning + function_call turn stays one assistant turn downstream.
+/// Flushes pending function calls and matching outputs while preserving adjacency.
+/// Pending reasoning rides in the same assistant message as the tool calls so a
+/// Codex reasoning + function_call turn stays one assistant turn downstream.
fn flush_responses_tool_block(
messages: &mut Vec<Message>,
pending_tool_calls: &mut Vec<ToolCall>,
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 56ae8d6ea..d17fa7f99 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -732,13 +732,18 @@ fn responses_unknown_input_item_is_preserved_for_openai_chat() -> TestResult {
Ok(())
}
-// Responses accepts message-shaped input items without an explicit discriminator.
+// Responses accepts message-shaped input items without an explicit discriminator, and inline
+// system and developer items keep their roles instead of being demoted to user.
#[test]
-fn responses_input_message_without_type_translates_normally() -> TestResult {
+fn responses_input_messages_translate_with_instruction_roles_intact() -> TestResult {
let engine = TranslationEngine::default();
let body = json!({
"model": "gpt-4",
- "input": [{"role": "user", "content": "hello"}]
+ "input": [
+ {"type": "message", "role": "system", "content": "Be terse."},
+ {"type": "message", "role": "developer", "content": "Return JSON only."},
+ {"role": "user", "content": "hello"}
+ ]
});
let output = engine
@@ -752,11 +757,46 @@ fn responses_input_message_without_type_translates_normally() -> TestResult {
assert_eq!(
output["messages"],
- json!([{"role": "user", "content": "hello"}])
+ json!([
+ {"role": "system", "content": "Be terse."},
+ {"role": "developer", "content": "Return JSON only."},
+ {"role": "user", "content": "hello"}
+ ])
);
Ok(())
}
+// Inline instruction items must not detach pending reasoning from the assistant
+// turn that produced it.
+#[test]
+fn responses_inline_instruction_does_not_detach_reasoning() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-4",
+ "input": [
+ {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "thinking..."}], "summary": []},
+ {"type": "message", "role": "system", "content": "Be terse."},
+ {"type": "message", "role": "assistant", "content": "hello"}
+ ]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(output["messages"].as_array().map(Vec::len), Some(2));
+ assert_eq!(output["messages"][0]["role"], "system");
+ assert_eq!(output["messages"][1]["role"], "assistant");
+ assert_eq!(output["messages"][1]["content"], "hello");
+ assert_eq!(output["messages"][1]["reasoning"], "thinking...");
+ Ok(())
+}
+
// A discriminator-less object that is not message-shaped must not silently become prompt text.
#[test]
fn responses_input_without_type_or_message_shape_is_rejected() {