Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(llm-client): drop reasoning items OpenAI cannot read from Responses requests (#1063)

Elyas Mehtabuddin42b6fca680cc
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 3e4fc675a..0265faf99 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -1027,21 +1027,28 @@ fn set_json_model(body: &mut Value, model: &str) {
const CODEX_NAMESPACE_SEPARATOR: &str = "__";
-// Codex extends Responses with namespace containers and namespaced function
-// calls. OpenAI-compatible providers expect a flat Responses tool namespace.
+/// Rewrites a Responses body into the shape OpenAI-compatible providers accept.
+///
+/// Codex extends Responses with namespace containers and namespaced function calls, and
+/// OpenAI-compatible providers expect a flat Responses tool namespace. The input also loses
+/// reasoning items that the provider cannot read.
fn sanitize_openai_responses_provider_body(body: &mut Value) {
let Value::Object(object) = body else {
return;
};
- sanitize_openai_responses_input_for_provider(object.get_mut("input"));
+ let store_off = object.get("store") == Some(&Value::Bool(false));
+ sanitize_openai_responses_input_for_provider(object.get_mut("input"), store_off);
sanitize_openai_responses_tools_for_provider(object.get_mut("tools"));
sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice"));
}
-fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {
+/// Removes reasoning items that the provider cannot read and flattens namespaced function
+/// calls in the Responses `input` array.
+fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>, store_off: bool) {
let Some(Value::Array(items)) = input else {
return;
};
+ items.retain(|item| !is_unreadable_reasoning_item(item, store_off));
for item in items {
let Some(object) = item.as_object_mut() else {
continue;
@@ -1052,6 +1059,28 @@ fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {
}
}
+/// Returns true for a reasoning item that an OpenAI provider would reject with 400.
+///
+/// A router can serve earlier steps of a turn from Claude and a later step from an OpenAI
+/// model, and the client replays Claude's reasoning items on every step. Two kinds fail:
+///
+/// - An item without `encrypted_content` (absent, null, or empty) when the request sets
+/// `store: false`. OpenAI treats the item's ID as a reference to stored reasoning and finds
+/// nothing. With `store` on, the ID can refer to stored reasoning, so the item is kept.
+/// - An item whose `encrypted_content` holds Anthropic thinking that Switchyard packed for a
+/// Responses client. OpenAI cannot decrypt it, whatever `store` is set to.
+fn is_unreadable_reasoning_item(item: &Value, store_off: bool) -> bool {
+ if item.get("type").and_then(Value::as_str) != Some("reasoning") {
+ return false;
+ }
+ match item.get("encrypted_content").and_then(Value::as_str) {
+ Some(encrypted) if !encrypted.is_empty() => {
+ switchyard_translation::is_anthropic_thinking_envelope(encrypted)
+ }
+ _ => store_off,
+ }
+}
+
fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) {
let Some(Value::Array(tools)) = tools else {
return;
@@ -3414,6 +3443,130 @@ mod tests {
Ok(())
}
+ // A stage route can answer early steps of a Codex turn from Claude and a later step from
+ // GPT, and Codex replays Claude's reasoning items on every step. OpenAI returns 400 for two
+ // kinds: items with an ID but no `encrypted_content` when the request sets `store: false`,
+ // and items whose `encrypted_content` holds Anthropic thinking that Switchyard packed for
+ // the client. The Responses leg must drop both and keep GPT's own encrypted reasoning.
+ #[tokio::test]
+ async fn openai_responses_backend_drops_reasoning_the_provider_cannot_read()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/responses"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "resp_1",
+ "model": "gpt",
+ "object": "response",
+ "created_at": 0,
+ "status": "completed",
+ "output": [{
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": "ok"}]
+ }],
+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?;
+ let claude_reasoning = json!({
+ "type": "reasoning",
+ "id": "rs_b99feb9a-763d-4f7f-b6b2-2c0199b47ded",
+ "summary": [{"type": "summary_text", "text": "Read the files first."}],
+ "encrypted_content": null
+ });
+ // Build the item the way Switchyard does when it returns a Claude reply to a Responses
+ // client, so the test follows any change to that format.
+ let claude_reply = switchyard_translation::TranslationEngine::default()
+ .translate_response(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiResponses,
+ &json!({
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude",
+ "content": [
+ {"type": "thinking", "thinking": "Plan the edit.", "signature": "sig-abc"},
+ {"type": "text", "text": "ok"}
+ ],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1}
+ }),
+ &switchyard_translation::TranslationPolicy::default(),
+ )?
+ .body;
+ let switchyard_reasoning = claude_reply["output"]
+ .as_array()
+ .and_then(|output| output.iter().find(|item| item["type"] == "reasoning"))
+ .cloned()
+ .ok_or("translated Claude reply should contain a reasoning item")?;
+ assert!(switchyard_translation::is_anthropic_thinking_envelope(
+ switchyard_reasoning["encrypted_content"]
+ .as_str()
+ .unwrap_or_default()
+ ));
+ let gpt_reasoning = json!({
+ "type": "reasoning",
+ "id": "rs_gpt",
+ "summary": [],
+ "encrypted_content": "opaque-encrypted-reasoning"
+ });
+ let empty_reasoning =
+ json!({"type": "reasoning", "id": "rs_empty", "summary": [], "encrypted_content": ""});
+ let user = json!({"type": "message", "role": "user", "content": "fix the build"});
+ let call = json!({"type": "function_call", "call_id": "call_1", "name": "exec", "arguments": "{}"});
+ let output = json!({"type": "function_call_output", "call_id": "call_1", "output": "ok"});
+ let input = json!([
+ user,
+ claude_reasoning,
+ switchyard_reasoning,
+ call,
+ output,
+ gpt_reasoning,
+ empty_reasoning
+ ]);
+
+ for store in [json!(false), json!(true)] {
+ let raw = json!({"model": "client-facing", "store": store, "input": input.clone()});
+ client
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiResponses,
+ )
+ .await?;
+ }
+
+ let received = server
+ .received_requests()
+ .await
+ .ok_or("request recording should be enabled")?;
+ let sent: Vec<Value> = received
+ .iter()
+ .map(|request| serde_json::from_slice::<Value>(&request.body))
+ .collect::<std::result::Result<_, _>>()?;
+ // With `store: false`, only GPT's own encrypted reasoning survives.
+ assert_eq!(sent[0]["input"], json!([user, call, output, gpt_reasoning]));
+ // With `store: true`, IDs can refer to stored reasoning, but OpenAI still cannot
+ // decrypt the Switchyard item.
+ assert_eq!(
+ sent[1]["input"],
+ json!([
+ user,
+ claude_reasoning,
+ call,
+ output,
+ gpt_reasoning,
+ empty_reasoning
+ ])
+ );
+ Ok(())
+ }
+
// Raw path, streaming: an inbound `stream: true` request yields an unframed stream
// of OpenAI Chat chunk objects whose deltas reassemble the completion.
#[tokio::test]
diff --git a/crates/switchyard-translation/src/codecs/responses/mod.rs b/crates/switchyard-translation/src/codecs/responses/mod.rs
index 3939a6c8c..8f18e26fb 100644
--- a/crates/switchyard-translation/src/codecs/responses/mod.rs
+++ b/crates/switchyard-translation/src/codecs/responses/mod.rs
@@ -69,3 +69,10 @@ fn encode_anthropic_thinking(text: &str, signature: &str) -> String {
fn decode_anthropic_thinking(payload: &str) -> Option<(String, String)> {
serde_json::from_str(payload.strip_prefix(ANTHROPIC_THINKING_PREFIX)?).ok()
}
+
+/// Returns true when a Responses `encrypted_content` value holds Anthropic thinking that
+/// Switchyard packed for a Responses client, rather than ciphertext from an OpenAI provider.
+/// Only Switchyard can read this value, so an OpenAI provider cannot decrypt it.
+pub fn is_anthropic_thinking_envelope(encrypted_content: &str) -> bool {
+ encrypted_content.starts_with(ANTHROPIC_THINKING_PREFIX)
+}
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 4d0e81da6..1285ffdc6 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -19,6 +19,7 @@ mod sse;
pub mod stream;
pub mod util;
+pub use codecs::responses::is_anthropic_thinking_envelope;
pub use switchyard_protocol::stream::{
LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError,
ProviderStreamEvent,