Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: add omit_body_fields config (#852)

Greg Clark66bac2908694
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index 1e3784a76..064b13546 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -4,7 +4,10 @@
//! Per-provider backend configuration: wire format, upstream URL, and auth.
use std::time::Duration;
-use std::{collections::BTreeMap, fmt};
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ fmt,
+};
use reqwest::RequestBuilder;
use reqwest::header::{HeaderName, HeaderValue};
@@ -59,6 +62,10 @@ pub struct HttpBackendConfig {
pub extra_headers: BTreeMap<String, String>,
/// Default top-level request fields, applied only when the request omits the key.
pub extra_body: BTreeMap<String, Value>,
+ /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
+ /// so a target can reinstate one deliberately. For providers that reject an otherwise
+ /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
+ pub omit_body_fields: BTreeSet<String>,
/// Reasoning effort forced on every request to this backend, replacing whatever the caller
/// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
/// Anthropic has no equivalent and rejects the setting at configuration time.
@@ -78,6 +85,7 @@ impl fmt::Debug for HttpBackendConfig {
.field("forward_auth", &self.forward_auth)
.field("extra_header_names", &self.extra_headers.keys())
.field("extra_body_keys", &self.extra_body.keys())
+ .field("omit_body_fields", &self.omit_body_fields)
.field("reasoning_effort", &self.reasoning_effort)
.field("max_retries", &self.max_retries)
.field("timeout", &self.timeout)
@@ -296,6 +304,11 @@ impl Backend {
&self.config().extra_body
}
+ /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
+ pub fn omit_body_fields(&self) -> &BTreeSet<String> {
+ &self.config().omit_body_fields
+ }
+
/// Reasoning effort forced on outbound requests, if the target configures one.
pub fn reasoning_effort(&self) -> Option<&str> {
self.config().reasoning_effort.as_deref()
@@ -415,6 +428,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 0,
timeout: None,
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index b32e5e586..f2d0301dd 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -4,7 +4,7 @@
//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral
//! request, call the configured backend over HTTP, decode the neutral response.
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::future::ready;
use std::time::{Duration, SystemTime};
@@ -260,6 +260,7 @@ impl TranslatingLlmClient {
strip_anthropic_incompatible_fields(&mut body);
strip_unsigned_thinking_blocks(&mut body);
}
+ omit_configured_body_fields(&mut body, backend.omit_body_fields());
merge_extra_body(&mut body, backend.extra_body());
// After the merge on purpose: the effort override must win over both the caller's
// value and any `reasoning` default a target set through `extra_body`.
@@ -1145,6 +1146,15 @@ fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
}
}
+fn omit_configured_body_fields(body: &mut Value, omit_body_fields: &BTreeSet<String>) {
+ let Value::Object(object) = body else {
+ return;
+ };
+ for key in omit_body_fields {
+ object.remove(key);
+ }
+}
+
// Anthropic and Bedrock both cap a request at four blocks carrying
// `cache_control`, counting tools, system blocks and message blocks together.
const MAX_CACHE_CONTROL_BLOCKS: usize = 4;
@@ -1294,6 +1304,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 0,
timeout: None,
@@ -1402,6 +1413,15 @@ mod tests {
vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
}
+ fn chat_map_with_omit_body_fields(
+ base_url: &str,
+ omit_body_fields: BTreeSet<String>,
+ ) -> Vec<ModelConfig> {
+ let mut backend = config(base_url);
+ backend.omit_body_fields = omit_body_fields;
+ vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
+ }
+
fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
let mut backend = config(base_url);
backend.reasoning_effort = Some(effort.to_string());
@@ -2054,6 +2074,50 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn omit_body_fields_strips_a_field_extra_body_cannot_override()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(|request: &wiremock::Request| {
+ let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null);
+ body.get("max_tokens").is_none()
+ })
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop"
+ }],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map_with_omit_body_fields(
+ &format!("{}/v1", server.uri()),
+ BTreeSet::from(["max_tokens".to_string()]),
+ ))?;
+ let raw = json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 7
+ });
+
+ client
+ .call_rewrite_model_raw(
+ raw,
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
+ .await?;
+ Ok(())
+ }
+
// A weak OpenAI-format tier emits thinking blocks with no signature. Replaying
// them to Anthropic is rejected (Bedrock reports it as a SigV4 mismatch), so
// the Anthropic leg must drop them while keeping signed ones.
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 73a541f1c..691e169cf 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -902,7 +902,7 @@ impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {
#[cfg(test)]
mod tests {
- use std::collections::BTreeMap;
+ use std::collections::{BTreeMap, BTreeSet};
use super::*;
use async_trait::async_trait;
@@ -1210,6 +1210,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::from([("store".to_string(), json!(store))]),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 0,
timeout: None,
@@ -1355,6 +1356,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 0,
timeout: None,
@@ -2026,6 +2028,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 2,
timeout: None,
@@ -2124,6 +2127,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries: 0,
timeout: None,
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 6da7a4f28..05f03e2ce 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -3,7 +3,7 @@
//! Version-1 TOML deployment loading for the shared runner.
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs;
use std::path::Path;
use std::sync::Arc;
@@ -193,9 +193,10 @@ impl DeploymentConfig {
let (first_name, first) = slot.get();
if first.reasoning_effort != target.reasoning_effort
|| first.extra_body != target.extra_body
+ || first.omit_body_fields != target.omit_body_fields
{
return Err(RunnerError::configuration(format!(
- "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort or extra_body; one target per model id is kept, so give each its own model id or llm client",
+ "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort, extra_body, or omit_body_fields; one target per model id is kept, so give each its own model id or llm client",
target.id, target.llm_client
)));
}
@@ -280,7 +281,13 @@ impl DeploymentConfig {
for (name, client_config) in &self.llm_clients {
validate_value("llm client name", name)?;
- let backend = build_backend(name, client_config, &BTreeMap::new(), None)?;
+ let backend = build_backend(
+ name,
+ client_config,
+ &BTreeMap::new(),
+ &BTreeSet::new(),
+ None,
+ )?;
let (Backend::OpenAiChat(config)
| Backend::OpenAiResponses(config)
| Backend::Anthropic(config)) = backend;
@@ -318,6 +325,7 @@ impl DeploymentConfig {
&target.llm_client,
client_config,
&target.extra_body,
+ &target.omit_body_fields,
target.reasoning_effort.clone(),
)?,
None,
@@ -579,6 +587,10 @@ struct TargetConfig {
llm_client: String,
#[serde(default)]
extra_body: BTreeMap<String, Value>,
+ /// Top-level request fields dropped from the outbound body before `extra_body` is merged.
+ /// For providers that reject an otherwise standard field.
+ #[serde(default)]
+ omit_body_fields: BTreeSet<String>,
system_prompt: Option<String>,
/// Reasoning effort forced on every request to this target, replacing the caller's value.
/// Only meaningful on `openai_chat` and `openai_responses` clients.
@@ -639,6 +651,7 @@ fn build_backend(
client_name: &str,
config: &LlmClientConfig,
extra_body: &BTreeMap<String, Value>,
+ omit_body_fields: &BTreeSet<String>,
reasoning_effort: Option<String>,
) -> RunnerResult<Backend> {
if config.max_retries > MAX_CONFIGURED_RETRIES {
@@ -684,6 +697,7 @@ fn build_backend(
forward_auth: config.forward_auth,
extra_headers: config.extra_headers.clone(),
extra_body: extra_body.clone(),
+ omit_body_fields: omit_body_fields.clone(),
reasoning_effort,
max_retries: config.max_retries,
timeout: config.timeout_ms.map(Duration::from_millis),
@@ -1194,7 +1208,8 @@ new = ["send_message"]
),
);
assert!(
- error_message(&conflicting).contains("different reasoning_effort or extra_body"),
+ error_message(&conflicting)
+ .contains("different reasoning_effort, extra_body, or omit_body_fields"),
"{}",
error_message(&conflicting)
);
@@ -1625,7 +1640,13 @@ confidence_threshold = 0.5
let Some(client) = config.llm_clients.get("primary") else {
return Err(RunnerError::configuration("primary llm client is missing"));
};
- let backend = build_backend("primary", client, &target.extra_body, None)?;
+ let backend = build_backend(
+ "primary",
+ client,
+ &target.extra_body,
+ &target.omit_body_fields,
+ None,
+ )?;
assert_eq!(
backend.extra_body().get("service_tier"),
@@ -1641,6 +1662,35 @@ confidence_threshold = 0.5
Ok(())
}
+ #[test]
+ fn target_omit_body_fields_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {
+ let configured = VALID_CONFIG.replacen(
+ "llm_client = \"primary\"",
+ "llm_client = \"primary\"\n\
+ omit_body_fields = [\"max_output_tokens\"]",
+ 1,
+ );
+ let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse config: {error}"))
+ })?;
+ let Some(target) = config.targets.get("classifier") else {
+ return Err(RunnerError::configuration("classifier target is missing"));
+ };
+ let Some(client) = config.llm_clients.get("primary") else {
+ return Err(RunnerError::configuration("primary llm client is missing"));
+ };
+ let backend = build_backend(
+ "primary",
+ client,
+ &target.extra_body,
+ &target.omit_body_fields,
+ None,
+ )?;
+
+ assert!(backend.omit_body_fields().contains("max_output_tokens"));
+ Ok(())
+ }
+
#[test]
fn client_deadline_defaults_and_rejects_zero() -> RunnerResult<()> {
for (setting, expected) in [
@@ -1652,7 +1702,7 @@ confidence_threshold = 0.5
"format = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\n{setting}"
);
let config: LlmClientConfig = toml::from_str(&source).expect("valid deadline config");
- let backend = build_backend("test", &config, &BTreeMap::new(), None);
+ let backend = build_backend("test", &config, &BTreeMap::new(), &BTreeSet::new(), None);
if expected == Some(0) {
assert!(backend.is_err());
} else {
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index dae5c1ad0..cd21d3c91 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -3,7 +3,7 @@
//! Integration tests for the libsy Rust server.
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::convert::Infallible;
use std::error::Error;
use std::io::Write;
@@ -749,6 +749,7 @@ fn random_state_with_retries(
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ omit_body_fields: BTreeSet::new(),
reasoning_effort: None,
max_retries,
timeout: None,
diff --git a/examples/run_codex.sh b/examples/run_codex.sh
new file mode 100644
index 000000000..d019301a2
--- /dev/null
+++ b/examples/run_codex.sh
@@ -0,0 +1,91 @@
+# Local Setup with Codex + Switchyard: composite router (Terra + Stage, Sol/Luna)
+#
+# Terra classifies the user's turn and sets the tier. Stage drives the tool
+# loop underneath it. Everything runs on your own ChatGPT login, no API key.
+#
+# Prereqs: Rust/Cargo, Codex CLI already logged in.
+
+cargo build -p switchyard-server --release
+
+cat > /tmp/composite.toml <<'EOF'
+schema_version = 1
+
+[llm_clients.chatgpt_backend]
+format = "openai_responses"
+base_url = "https://chatgpt.com/backend-api/codex"
+forward_auth = true
+
+[targets.capable]
+id = "gpt-5.6-sol"
+llm_client = "chatgpt_backend"
+
+[targets.efficient]
+id = "gpt-5.6-luna"
+llm_client = "chatgpt_backend"
+
+# chatgpt.com/backend-api/codex is Codex CLI's own private endpoint, not the
+# public OpenAI Responses API. It 400s unless store=false and stream=true are
+# set explicitly, and it rejects max_output_tokens outright, so the
+# classifier's own token cap has to be dropped before the request goes out.
+[targets.terra]
+id = "gpt-5.6-terra"
+llm_client = "chatgpt_backend"
+extra_body = { store = false, stream = true }
+omit_body_fields = ["max_output_tokens"]
+
+[routes.switchyard]
+id = "switchyard"
+type = "composite"
+
+[routes.switchyard.classifier]
+target = "terra"
+base_threshold = 0.5
+classify_trigger = "user_turn"
+
+[routes.switchyard.stage]
+capable_target = "capable"
+efficient_target = "efficient"
+confidence_threshold = 0.5
+EOF
+
+./target/release/switchyard-server --config /tmp/composite.toml --dry-run
+./target/release/switchyard-server --config /tmp/composite.toml --port 4123 &
+
+cat > ~/.codex/composite_test.config.toml <<'EOF'
+model = "switchyard"
+model_provider = "sy_composite"
+approval_policy = "never"
+sandbox_mode = "workspace-write"
+
+[model_providers.sy_composite]
+name = "Switchyard Composite"
+base_url = "http://127.0.0.1:4123/v1"
+wire_api = "responses"
+requires_openai_auth = true
+EOF
+
+codex --profile composite_test
+
+# What's happening
+#
+# Terra reads the user's turn once and sets the tier, held for every tool
+# call until the next user turn. Stage's own signals still run underneath
+# and can escalate mid-turn on a genuine failure, they just start from
+# Terra's tier instead of the picker's static default.
+#
+# Verify
+#
+# Check the server's log or curl -s http://127.0.0.1:4123/v1/stats.
+# selected_model should follow Terra's read of the prompt, not just default
+# to Luna, and a tool failure mid-turn should still push it to Sol.
+#
+# Known quirks
+#
+# chatgpt.com/backend-api/codex is Codex CLI's own private endpoint, not the
+# public OpenAI Responses API, and it only accepts requests shaped the way
+# the real Codex CLI sends them: store=false and stream=true set explicitly,
+# and no max_output_tokens field at all (it 400s on that one outright). The
+# terra target's extra_body/omit_body_fields above exist to make the
+# classifier's request match that shape; capable/efficient don't need them
+# because they forward the caller's own Responses body verbatim.
+