Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: add omit_body_fields config (#852)
Greg Clark66bac2908694
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex 1e3784a76..064b13546 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -4,7 +4,10 @@//! Per-provider backend configuration: wire format, upstream URL, and auth.use std::time::Duration;-use std::{collections::BTreeMap, fmt};+use std::{+ collections::{BTreeMap, BTreeSet},+ fmt,+};use reqwest::RequestBuilder;use reqwest::header::{HeaderName, HeaderValue};@@ -59,6 +62,10 @@ pub struct HttpBackendConfig {pub extra_headers: BTreeMap<String, String>,/// Default top-level request fields, applied only when the request omits the key.pub extra_body: BTreeMap<String, Value>,+ /// Top-level request fields dropped from the outbound body before `extra_body` is merged,+ /// so a target can reinstate one deliberately. For providers that reject an otherwise+ /// standard field (e.g. an internal API that 400s on `max_output_tokens`).+ pub omit_body_fields: BTreeSet<String>,/// Reasoning effort forced on every request to this backend, replacing whatever the caller/// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;/// Anthropic has no equivalent and rejects the setting at configuration time.@@ -78,6 +85,7 @@ impl fmt::Debug for HttpBackendConfig {.field("forward_auth", &self.forward_auth).field("extra_header_names", &self.extra_headers.keys()).field("extra_body_keys", &self.extra_body.keys())+ .field("omit_body_fields", &self.omit_body_fields).field("reasoning_effort", &self.reasoning_effort).field("max_retries", &self.max_retries).field("timeout", &self.timeout)@@ -296,6 +304,11 @@ impl Backend {&self.config().extra_body}+ /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.+ pub fn omit_body_fields(&self) -> &BTreeSet<String> {+ &self.config().omit_body_fields+ }+/// Reasoning effort forced on outbound requests, if the target configures one.pub fn reasoning_effort(&self) -> Option<&str> {self.config().reasoning_effort.as_deref()@@ -415,6 +428,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 0,timeout: None,diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex b32e5e586..f2d0301dd 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -4,7 +4,7 @@//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral//! request, call the configured backend over HTTP, decode the neutral response.-use std::collections::{BTreeMap, HashMap};+use std::collections::{BTreeMap, BTreeSet, HashMap};use std::future::ready;use std::time::{Duration, SystemTime};@@ -260,6 +260,7 @@ impl TranslatingLlmClient {strip_anthropic_incompatible_fields(&mut body);strip_unsigned_thinking_blocks(&mut body);}+ omit_configured_body_fields(&mut body, backend.omit_body_fields());merge_extra_body(&mut body, backend.extra_body());// After the merge on purpose: the effort override must win over both the caller's// value and any `reasoning` default a target set through `extra_body`.@@ -1145,6 +1146,15 @@ fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {}}+fn omit_configured_body_fields(body: &mut Value, omit_body_fields: &BTreeSet<String>) {+ let Value::Object(object) = body else {+ return;+ };+ for key in omit_body_fields {+ object.remove(key);+ }+}+// Anthropic and Bedrock both cap a request at four blocks carrying// `cache_control`, counting tools, system blocks and message blocks together.const MAX_CACHE_CONTROL_BLOCKS: usize = 4;@@ -1294,6 +1304,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 0,timeout: None,@@ -1402,6 +1413,15 @@ mod tests {vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]}+ fn chat_map_with_omit_body_fields(+ base_url: &str,+ omit_body_fields: BTreeSet<String>,+ ) -> Vec<ModelConfig> {+ let mut backend = config(base_url);+ backend.omit_body_fields = omit_body_fields;+ vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]+ }+fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {let mut backend = config(base_url);backend.reasoning_effort = Some(effort.to_string());@@ -2054,6 +2074,50 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn omit_body_fields_strips_a_field_extra_body_cannot_override()+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(|request: &wiremock::Request| {+ let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);+ body.get("max_tokens").is_none()+ })+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop"+ }],+ "usage": {}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(+ &format!("{}/v1", server.uri()),+ BTreeSet::from(["max_tokens".to_string()]),+ ))?;+ let raw = json!({+ "model": "client-facing",+ "messages": [{"role": "user", "content": "hi"}],+ "max_tokens": 7+ });++ client+ .call_rewrite_model_raw(+ raw,+ None,+ Some(&ModelId::from("gpt")),+ WireFormat::OpenAiChat,+ )+ .await?;+ Ok(())+ }+// A weak OpenAI-format tier emits thinking blocks with no signature. Replaying// them to Anthropic is rejected (Bedrock reports it as a SigV4 mismatch), so// the Anthropic leg must drop them while keeping signed ones.diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 73a541f1c..691e169cf 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -902,7 +902,7 @@ impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {#[cfg(test)]mod tests {- use std::collections::BTreeMap;+ use std::collections::{BTreeMap, BTreeSet};use super::*;use async_trait::async_trait;@@ -1210,6 +1210,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::from([("store".to_string(), json!(store))]),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 0,timeout: None,@@ -1355,6 +1356,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 0,timeout: None,@@ -2026,6 +2028,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 2,timeout: None,@@ -2124,6 +2127,7 @@ mod tests {forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries: 0,timeout: None,diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 6da7a4f28..05f03e2ce 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -3,7 +3,7 @@//! Version-1 TOML deployment loading for the shared runner.-use std::collections::{BTreeMap, HashMap};+use std::collections::{BTreeMap, BTreeSet, HashMap};use std::fs;use std::path::Path;use std::sync::Arc;@@ -193,9 +193,10 @@ impl DeploymentConfig {let (first_name, first) = slot.get();if first.reasoning_effort != target.reasoning_effort|| first.extra_body != target.extra_body+ || first.omit_body_fields != target.omit_body_fields{return Err(RunnerError::configuration(format!(- "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort or extra_body; one target per model id is kept, so give each its own model id or llm client",+ "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, extra_body, or omit_body_fields; one target per model id is kept, so give each its own model id or llm client",target.id, target.llm_client)));}@@ -280,7 +281,13 @@ impl DeploymentConfig {for (name, client_config) in &self.llm_clients {validate_value("llm client name", name)?;- let backend = build_backend(name, client_config, &BTreeMap::new(), None)?;+ let backend = build_backend(+ name,+ client_config,+ &BTreeMap::new(),+ &BTreeSet::new(),+ None,+ )?;let (Backend::OpenAiChat(config)| Backend::OpenAiResponses(config)| Backend::Anthropic(config)) = backend;@@ -318,6 +325,7 @@ impl DeploymentConfig {&target.llm_client,client_config,&target.extra_body,+ &target.omit_body_fields,target.reasoning_effort.clone(),)?,None,@@ -579,6 +587,10 @@ struct TargetConfig {llm_client: String,#[serde(default)]extra_body: BTreeMap<String, Value>,+ /// Top-level request fields dropped from the outbound body before `extra_body` is merged.+ /// For providers that reject an otherwise standard field.+ #[serde(default)]+ omit_body_fields: BTreeSet<String>,system_prompt: Option<String>,/// Reasoning effort forced on every request to this target, replacing the caller's value./// Only meaningful on `openai_chat` and `openai_responses` clients.@@ -639,6 +651,7 @@ fn build_backend(client_name: &str,config: &LlmClientConfig,extra_body: &BTreeMap<String, Value>,+ omit_body_fields: &BTreeSet<String>,reasoning_effort: Option<String>,) -> RunnerResult<Backend> {if config.max_retries > MAX_CONFIGURED_RETRIES {@@ -684,6 +697,7 @@ fn build_backend(forward_auth: config.forward_auth,extra_headers: config.extra_headers.clone(),extra_body: extra_body.clone(),+ omit_body_fields: omit_body_fields.clone(),reasoning_effort,max_retries: config.max_retries,timeout: config.timeout_ms.map(Duration::from_millis),@@ -1194,7 +1208,8 @@ new = ["send_message"]),);assert!(- error_message(&conflicting).contains("different reasoning_effort or extra_body"),+ error_message(&conflicting)+ .contains("different reasoning_effort, extra_body, or omit_body_fields"),"{}",error_message(&conflicting));@@ -1625,7 +1640,13 @@ confidence_threshold = 0.5let Some(client) = config.llm_clients.get("primary") else {return Err(RunnerError::configuration("primary llm client is missing"));};- let backend = build_backend("primary", client, &target.extra_body, None)?;+ let backend = build_backend(+ "primary",+ client,+ &target.extra_body,+ &target.omit_body_fields,+ None,+ )?;assert_eq!(backend.extra_body().get("service_tier"),@@ -1641,6 +1662,35 @@ confidence_threshold = 0.5Ok(())}+ #[test]+ fn target_omit_body_fields_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {+ let configured = VALID_CONFIG.replacen(+ "llm_client = \"primary\"",+ "llm_client = \"primary\"\n\+ omit_body_fields = [\"max_output_tokens\"]",+ 1,+ );+ let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {+ RunnerError::configuration(format!("failed to parse config: {error}"))+ })?;+ let Some(target) = config.targets.get("classifier") else {+ return Err(RunnerError::configuration("classifier target is missing"));+ };+ let Some(client) = config.llm_clients.get("primary") else {+ return Err(RunnerError::configuration("primary llm client is missing"));+ };+ let backend = build_backend(+ "primary",+ client,+ &target.extra_body,+ &target.omit_body_fields,+ None,+ )?;++ assert!(backend.omit_body_fields().contains("max_output_tokens"));+ Ok(())+ }+#[test]fn client_deadline_defaults_and_rejects_zero() -> RunnerResult<()> {for (setting, expected) in [@@ -1652,7 +1702,7 @@ confidence_threshold = 0.5"format = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\n{setting}");let config: LlmClientConfig = toml::from_str(&source).expect("valid deadline config");- let backend = build_backend("test", &config, &BTreeMap::new(), None);+ let backend = build_backend("test", &config, &BTreeMap::new(), &BTreeSet::new(), None);if expected == Some(0) {assert!(backend.is_err());} else {diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex dae5c1ad0..cd21d3c91 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -3,7 +3,7 @@//! Integration tests for the libsy Rust server.-use std::collections::{BTreeMap, HashSet};+use std::collections::{BTreeMap, BTreeSet, HashSet};use std::convert::Infallible;use std::error::Error;use std::io::Write;@@ -749,6 +749,7 @@ fn random_state_with_retries(forward_auth: false,extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ omit_body_fields: BTreeSet::new(),reasoning_effort: None,max_retries,timeout: None,diff --git a/examples/run_codex.sh b/examples/run_codex.shnew file mode 100644index 000000000..d019301a2--- /dev/null+++ b/examples/run_codex.sh@@ -0,0 +1,91 @@+# Local Setup with Codex + Switchyard: composite router (Terra + Stage, Sol/Luna)+#+# Terra classifies the user's turn and sets the tier. Stage drives the tool+# loop underneath it. Everything runs on your own ChatGPT login, no API key.+#+# Prereqs: Rust/Cargo, Codex CLI already logged in.++cargo build -p switchyard-server --release++cat > /tmp/composite.toml <<'EOF'+schema_version = 1++[llm_clients.chatgpt_backend]+format = "openai_responses"+base_url = "https://chatgpt.com/backend-api/codex"+forward_auth = true++[targets.capable]+id = "gpt-5.6-sol"+llm_client = "chatgpt_backend"++[targets.efficient]+id = "gpt-5.6-luna"+llm_client = "chatgpt_backend"++# chatgpt.com/backend-api/codex is Codex CLI's own private endpoint, not the+# public OpenAI Responses API. It 400s unless store=false and stream=true are+# set explicitly, and it rejects max_output_tokens outright, so the+# classifier's own token cap has to be dropped before the request goes out.+[targets.terra]+id = "gpt-5.6-terra"+llm_client = "chatgpt_backend"+extra_body = { store = false, stream = true }+omit_body_fields = ["max_output_tokens"]++[routes.switchyard]+id = "switchyard"+type = "composite"++[routes.switchyard.classifier]+target = "terra"+base_threshold = 0.5+classify_trigger = "user_turn"++[routes.switchyard.stage]+capable_target = "capable"+efficient_target = "efficient"+confidence_threshold = 0.5+EOF++./target/release/switchyard-server --config /tmp/composite.toml --dry-run+./target/release/switchyard-server --config /tmp/composite.toml --port 4123 &++cat > ~/.codex/composite_test.config.toml <<'EOF'+model = "switchyard"+model_provider = "sy_composite"+approval_policy = "never"+sandbox_mode = "workspace-write"++[model_providers.sy_composite]+name = "Switchyard Composite"+base_url = "http://127.0.0.1:4123/v1"+wire_api = "responses"+requires_openai_auth = true+EOF++codex --profile composite_test++# What's happening+#+# Terra reads the user's turn once and sets the tier, held for every tool+# call until the next user turn. Stage's own signals still run underneath+# and can escalate mid-turn on a genuine failure, they just start from+# Terra's tier instead of the picker's static default.+#+# Verify+#+# Check the server's log or curl -s http://127.0.0.1:4123/v1/stats.+# selected_model should follow Terra's read of the prompt, not just default+# to Luna, and a tool failure mid-turn should still push it to Sol.+#+# Known quirks+#+# chatgpt.com/backend-api/codex is Codex CLI's own private endpoint, not the+# public OpenAI Responses API, and it only accepts requests shaped the way+# the real Codex CLI sends them: store=false and stream=true set explicitly,+# and no max_output_tokens field at all (it 400s on that one outright). The+# terra target's extra_body/omit_body_fields above exist to make the+# classifier's request match that shape; capable/efficient don't need them+# because they forward the caller's own Responses body verbatim.+