Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(llm-client): drop reasoning items OpenAI cannot read from Responses requests (#1063)
Elyas Mehtabuddin42b6fca680cc
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 3e4fc675a..0265faf99 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -1027,21 +1027,28 @@ fn set_json_model(body: &mut Value, model: &str) {const CODEX_NAMESPACE_SEPARATOR: &str = "__";-// Codex extends Responses with namespace containers and namespaced function-// calls. OpenAI-compatible providers expect a flat Responses tool namespace.+/// Rewrites a Responses body into the shape OpenAI-compatible providers accept.+///+/// Codex extends Responses with namespace containers and namespaced function calls, and+/// OpenAI-compatible providers expect a flat Responses tool namespace. The input also loses+/// reasoning items that the provider cannot read.fn sanitize_openai_responses_provider_body(body: &mut Value) {let Value::Object(object) = body else {return;};- sanitize_openai_responses_input_for_provider(object.get_mut("input"));+ let store_off = object.get("store") == Some(&Value::Bool(false));+ sanitize_openai_responses_input_for_provider(object.get_mut("input"), store_off);sanitize_openai_responses_tools_for_provider(object.get_mut("tools"));sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice"));}-fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {+/// Removes reasoning items that the provider cannot read and flattens namespaced function+/// calls in the Responses `input` array.+fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>, store_off: bool) {let Some(Value::Array(items)) = input else {return;};+ items.retain(|item| !is_unreadable_reasoning_item(item, store_off));for item in items {let Some(object) = item.as_object_mut() else {continue;@@ -1052,6 +1059,28 @@ fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) {}}+/// Returns true for a reasoning item that an OpenAI provider would reject with 400.+///+/// A router can serve earlier steps of a turn from Claude and a later step from an OpenAI+/// model, and the client replays Claude's reasoning items on every step. Two kinds fail:+///+/// - An item without `encrypted_content` (absent, null, or empty) when the request sets+/// `store: false`. OpenAI treats the item's ID as a reference to stored reasoning and finds+/// nothing. With `store` on, the ID can refer to stored reasoning, so the item is kept.+/// - An item whose `encrypted_content` holds Anthropic thinking that Switchyard packed for a+/// Responses client. OpenAI cannot decrypt it, whatever `store` is set to.+fn is_unreadable_reasoning_item(item: &Value, store_off: bool) -> bool {+ if item.get("type").and_then(Value::as_str) != Some("reasoning") {+ return false;+ }+ match item.get("encrypted_content").and_then(Value::as_str) {+ Some(encrypted) if !encrypted.is_empty() => {+ switchyard_translation::is_anthropic_thinking_envelope(encrypted)+ }+ _ => store_off,+ }+}+fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) {let Some(Value::Array(tools)) = tools else {return;@@ -3414,6 +3443,130 @@ mod tests {Ok(())}+ // A stage route can answer early steps of a Codex turn from Claude and a later step from+ // GPT, and Codex replays Claude's reasoning items on every step. OpenAI returns 400 for two+ // kinds: items with an ID but no `encrypted_content` when the request sets `store: false`,+ // and items whose `encrypted_content` holds Anthropic thinking that Switchyard packed for+ // the client. The Responses leg must drop both and keep GPT's own encrypted reasoning.+ #[tokio::test]+ async fn openai_responses_backend_drops_reasoning_the_provider_cannot_read()+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/responses"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "resp_1",+ "model": "gpt",+ "object": "response",+ "created_at": 0,+ "status": "completed",+ "output": [{+ "type": "message",+ "role": "assistant",+ "content": [{"type": "output_text", "text": "ok"}]+ }],+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?;+ let claude_reasoning = json!({+ "type": "reasoning",+ "id": "rs_b99feb9a-763d-4f7f-b6b2-2c0199b47ded",+ "summary": [{"type": "summary_text", "text": "Read the files first."}],+ "encrypted_content": null+ });+ // Build the item the way Switchyard does when it returns a Claude reply to a Responses+ // client, so the test follows any change to that format.+ let claude_reply = switchyard_translation::TranslationEngine::default()+ .translate_response(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiResponses,+ &json!({+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude",+ "content": [+ {"type": "thinking", "thinking": "Plan the edit.", "signature": "sig-abc"},+ {"type": "text", "text": "ok"}+ ],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 1, "output_tokens": 1}+ }),+ &switchyard_translation::TranslationPolicy::default(),+ )?+ .body;+ let switchyard_reasoning = claude_reply["output"]+ .as_array()+ .and_then(|output| output.iter().find(|item| item["type"] == "reasoning"))+ .cloned()+ .ok_or("translated Claude reply should contain a reasoning item")?;+ assert!(switchyard_translation::is_anthropic_thinking_envelope(+ switchyard_reasoning["encrypted_content"]+ .as_str()+ .unwrap_or_default()+ ));+ let gpt_reasoning = json!({+ "type": "reasoning",+ "id": "rs_gpt",+ "summary": [],+ "encrypted_content": "opaque-encrypted-reasoning"+ });+ let empty_reasoning =+ json!({"type": "reasoning", "id": "rs_empty", "summary": [], "encrypted_content": ""});+ let user = json!({"type": "message", "role": "user", "content": "fix the build"});+ let call = json!({"type": "function_call", "call_id": "call_1", "name": "exec", "arguments": "{}"});+ let output = json!({"type": "function_call_output", "call_id": "call_1", "output": "ok"});+ let input = json!([+ user,+ claude_reasoning,+ switchyard_reasoning,+ call,+ output,+ gpt_reasoning,+ empty_reasoning+ ]);++ for store in [json!(false), json!(true)] {+ let raw = json!({"model": "client-facing", "store": store, "input": input.clone()});+ client+ .call_rewrite_model_raw(+ raw,+ None,+ Some(&ModelId::from("gpt")),+ WireFormat::OpenAiResponses,+ )+ .await?;+ }++ let received = server+ .received_requests()+ .await+ .ok_or("request recording should be enabled")?;+ let sent: Vec<Value> = received+ .iter()+ .map(|request| serde_json::from_slice::<Value>(&request.body))+ .collect::<std::result::Result<_, _>>()?;+ // With `store: false`, only GPT's own encrypted reasoning survives.+ assert_eq!(sent[0]["input"], json!([user, call, output, gpt_reasoning]));+ // With `store: true`, IDs can refer to stored reasoning, but OpenAI still cannot+ // decrypt the Switchyard item.+ assert_eq!(+ sent[1]["input"],+ json!([+ user,+ claude_reasoning,+ call,+ output,+ gpt_reasoning,+ empty_reasoning+ ])+ );+ Ok(())+ }+// Raw path, streaming: an inbound `stream: true` request yields an unframed stream// of OpenAI Chat chunk objects whose deltas reassemble the completion.#[tokio::test]diff --git a/crates/switchyard-translation/src/codecs/responses/mod.rs b/crates/switchyard-translation/src/codecs/responses/mod.rsindex 3939a6c8c..8f18e26fb 100644--- a/crates/switchyard-translation/src/codecs/responses/mod.rs+++ b/crates/switchyard-translation/src/codecs/responses/mod.rs@@ -69,3 +69,10 @@ fn encode_anthropic_thinking(text: &str, signature: &str) -> String {fn decode_anthropic_thinking(payload: &str) -> Option<(String, String)> {serde_json::from_str(payload.strip_prefix(ANTHROPIC_THINKING_PREFIX)?).ok()}++/// Returns true when a Responses `encrypted_content` value holds Anthropic thinking that+/// Switchyard packed for a Responses client, rather than ciphertext from an OpenAI provider.+/// Only Switchyard can read this value, so an OpenAI provider cannot decrypt it.+pub fn is_anthropic_thinking_envelope(encrypted_content: &str) -> bool {+ encrypted_content.starts_with(ANTHROPIC_THINKING_PREFIX)+}diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rsindex 4d0e81da6..1285ffdc6 100644--- a/crates/switchyard-translation/src/lib.rs+++ b/crates/switchyard-translation/src/lib.rs@@ -19,6 +19,7 @@ mod sse;pub mod stream;pub mod util;+pub use codecs::responses::is_anthropic_thinking_envelope;pub use switchyard_protocol::stream::{LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError,ProviderStreamEvent,