Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): let a target force the reasoning effort of the requests it serves (#666)

Lin Jia832374ac4b97
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 025506956..995648431 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Added
+- **Per-target `reasoning_effort`** — a target can force the reasoning effort
+ of every request it serves, replacing the caller's value (`reasoning.effort`
+ on the Responses wire, `reasoning_effort` on Chat Completions), so a strong
+ tier can run at `max` behind a client that sends `high`. `extra_body` only
+ fills absent keys and could not do this. Rejected on Anthropic clients.
- **Raw Responses stream trace** — an opt-in trace of every upstream Responses
event as received, under `RUST_LOG=switchyard_translation::responses::raw=trace`,
for diagnosing provider-specific event shapes. (#646)
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index a526712fc..71f5d281d 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -54,6 +54,10 @@ pub struct HttpBackendConfig {
pub extra_headers: BTreeMap<String, String>,
/// Default top-level request fields, applied only when the request omits the key.
pub extra_body: BTreeMap<String, Value>,
+ /// Reasoning effort forced on every request to this backend, replacing whatever the caller
+ /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`;
+ /// Anthropic has no equivalent and rejects the setting at configuration time.
+ pub reasoning_effort: Option<String>,
/// Additional attempts after the initial upstream request.
pub max_retries: u32,
}
@@ -66,6 +70,7 @@ impl fmt::Debug for HttpBackendConfig {
.field("forward_auth", &self.forward_auth)
.field("extra_header_names", &self.extra_headers.keys())
.field("extra_body_keys", &self.extra_body.keys())
+ .field("reasoning_effort", &self.reasoning_effort)
.field("max_retries", &self.max_retries)
.finish()
}
@@ -250,6 +255,11 @@ impl Backend {
&self.config().extra_body
}
+ /// Reasoning effort forced on outbound requests, if the target configures one.
+ pub fn reasoning_effort(&self) -> Option<&str> {
+ self.config().reasoning_effort.as_deref()
+ }
+
/// Additional attempts allowed after the initial request.
pub fn max_retries(&self) -> u32 {
self.config().max_retries
@@ -345,6 +355,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
max_retries: 0,
}
}
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 6cecd7084..2dacc6383 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -256,6 +256,9 @@ impl TranslatingLlmClient {
strip_unsigned_thinking_blocks(&mut body);
}
merge_extra_body(&mut body, backend.extra_body());
+ // After the merge on purpose: the effort override must win over both the caller's
+ // value and any `reasoning` default a target set through `extra_body`.
+ apply_reasoning_effort(&mut body, backend);
if matches!(backend, Backend::Anthropic(_)) {
enable_anthropic_prompt_caching(&mut body);
}
@@ -933,6 +936,44 @@ fn is_unsigned_thinking_block(block: &Value) -> bool {
)
}
+// Forces the target's configured reasoning effort onto the outbound body, replacing the
+// caller's value. Unlike `extra_body`, this is an override: a route that sends one model at a
+// higher effort than the client asked for is the point of the setting.
+fn apply_reasoning_effort(body: &mut Value, backend: &Backend) {
+ let Some(effort) = backend.reasoning_effort() else {
+ return;
+ };
+ let Value::Object(object) = body else {
+ return;
+ };
+ match backend {
+ Backend::OpenAiResponses(_) => {
+ // Responses nests effort under `reasoning` next to fields the caller may have set
+ // (`summary`, for example), so only the `effort` key is replaced. A `reasoning`
+ // value that is not an object is malformed and is replaced whole.
+ let reasoning = object
+ .entry("reasoning".to_string())
+ .or_insert_with(|| Value::Object(serde_json::Map::new()));
+ if !reasoning.is_object() {
+ *reasoning = Value::Object(serde_json::Map::new());
+ }
+ if let Value::Object(reasoning) = reasoning {
+ reasoning.insert("effort".to_string(), Value::String(effort.to_string()));
+ }
+ }
+ Backend::OpenAiChat(_) => {
+ // Chat Completions takes effort as a top-level field.
+ object.insert(
+ "reasoning_effort".to_string(),
+ Value::String(effort.to_string()),
+ );
+ }
+ // Anthropic has no effort field (thinking is a token budget); the runner rejects the
+ // setting on Anthropic clients at load time, so this arm is unreachable in practice.
+ Backend::Anthropic(_) => {}
+ }
+}
+
// Applies target defaults without overriding fields supplied by the caller.
fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
let Value::Object(object) = body else {
@@ -1051,6 +1092,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
max_retries: 0,
}
}
@@ -1088,6 +1130,22 @@ mod tests {
vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
}
+ fn chat_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
+ let mut backend = config(base_url);
+ backend.reasoning_effort = Some(effort.to_string());
+ vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
+ }
+
+ fn responses_map_with_effort(base_url: &str, effort: &str) -> Vec<ModelConfig> {
+ let mut backend = config(base_url);
+ backend.reasoning_effort = Some(effort.to_string());
+ vec![ModelConfig::new(
+ "gpt",
+ Backend::OpenAiResponses(backend),
+ None,
+ )]
+ }
+
fn anthropic_map(base_url: &str) -> Vec<ModelConfig> {
vec![ModelConfig::new(
"claude",
@@ -1541,6 +1599,89 @@ mod tests {
Ok(())
}
+ /// A configured reasoning effort replaces the caller's value on both OpenAI wire formats,
+ /// which `extra_body` (defaults only) cannot do.
+ #[tokio::test]
+ async fn reasoning_effort_override_replaces_the_callers_effort()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "model": "gpt",
+ "reasoning_effort": "max"
+ })))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop"
+ }],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+ let client = TranslatingLlmClient::new(&chat_map_with_effort(
+ &format!("{}/v1", server.uri()),
+ "max",
+ ))?;
+ client
+ .call_rewrite_model_raw(
+ json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "reasoning_effort": "high"
+ }),
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiChat,
+ )
+ .await?;
+
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/responses"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "model": "gpt",
+ "reasoning": {"effort": "max", "summary": "auto"}
+ })))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "resp_1",
+ "object": "response",
+ "model": "gpt",
+ "status": "completed",
+ "output": [{
+ "type": "message",
+ "id": "msg_1",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": "ok"}]
+ }],
+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
+ })))
+ .mount(&server)
+ .await;
+ let client = TranslatingLlmClient::new(&responses_map_with_effort(
+ &format!("{}/v1", server.uri()),
+ "max",
+ ))?;
+ client
+ .call_rewrite_model_raw(
+ json!({
+ "model": "client-facing",
+ "input": [{"role": "user", "content": "hi"}],
+ "reasoning": {"effort": "high", "summary": "auto"}
+ }),
+ None,
+ Some(&ModelId::from("gpt")),
+ WireFormat::OpenAiResponses,
+ )
+ .await?;
+ Ok(())
+ }
+
#[tokio::test]
async fn extra_body_adds_defaults_without_overriding_the_request()
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 8561816e7..9a98ea614 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -894,6 +894,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
max_retries: 2,
})
};
@@ -985,6 +986,7 @@ mod tests {
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
max_retries: 0,
})
};
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index a2ef9aea7..867a8e53a 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -3,7 +3,7 @@
//! Version-1 TOML deployment loading for the shared runner.
-use std::collections::{BTreeMap, HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap};
use std::fs;
use std::path::Path;
use std::sync::Arc;
@@ -160,16 +160,35 @@ impl DeploymentConfig {
)));
}
- let mut seen_client_model_ids = HashSet::new();
+ // The LLM client keeps one backend per model id, so two targets naming the same model on
+ // the same client share it. That is harmless when their request settings agree (an alias
+ // for a different system prompt, say) and silently wrong when they do not: the second
+ // target's reasoning_effort or extra_body would never reach the wire.
+ let mut seen_client_model_ids: HashMap<(&str, &str), (&String, &TargetConfig)> =
+ HashMap::new();
for (target_name, target) in &self.targets {
validate_value("target name", target_name)?;
validate_value(&format!("target {target_name} id"), &target.id)?;
- if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) {
- tracing::warn!(
- "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",
- target.id,
- target.llm_client
- );
+ match seen_client_model_ids.entry((target.llm_client.as_str(), target.id.as_str())) {
+ std::collections::hash_map::Entry::Vacant(slot) => {
+ slot.insert((target_name, target));
+ }
+ std::collections::hash_map::Entry::Occupied(slot) => {
+ let (first_name, first) = slot.get();
+ if first.reasoning_effort != target.reasoning_effort
+ || first.extra_body != target.extra_body
+ {
+ return Err(RunnerError::configuration(format!(
+ "targets {first_name} and {target_name} both name model {} on llm client {} but with different reasoning_effort or extra_body; one target per model id is kept, so give each its own model id or llm client",
+ target.id, target.llm_client
+ )));
+ }
+ tracing::warn!(
+ "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",
+ target.id,
+ target.llm_client
+ );
+ }
}
}
@@ -232,7 +251,7 @@ impl DeploymentConfig {
for (name, client_config) in &self.llm_clients {
validate_value("llm client name", name)?;
- build_backend(name, client_config, &BTreeMap::new())?;
+ build_backend(name, client_config, &BTreeMap::new(), None)?;
}
for (target_name, target) in &self.targets {
let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
@@ -246,9 +265,26 @@ impl DeploymentConfig {
.ok_or_else(|| {
RunnerError::configuration("validated llm client was not initialized")
})?;
+ if let Some(effort) = &target.reasoning_effort {
+ if effort.trim().is_empty() {
+ return Err(RunnerError::configuration(format!(
+ "target {target_name} reasoning_effort must not be empty"
+ )));
+ }
+ if matches!(client_config.format, ClientFormat::AnthropicMessages) {
+ return Err(RunnerError::configuration(format!(
+ "target {target_name} reasoning_effort is only supported on openai_chat and openai_responses clients"
+ )));
+ }
+ }
model_configs.push(ModelConfig::new(
target.id.clone(),
- build_backend(&target.llm_client, client_config, &target.extra_body)?,
+ build_backend(
+ &target.llm_client,
+ client_config,
+ &target.extra_body,
+ target.reasoning_effort.clone(),
+ )?,
None,
));
}
@@ -489,6 +525,9 @@ struct TargetConfig {
#[serde(default)]
extra_body: BTreeMap<String, Value>,
system_prompt: Option<String>,
+ /// Reasoning effort forced on every request to this target, replacing the caller's value.
+ /// Only meaningful on `openai_chat` and `openai_responses` clients.
+ reasoning_effort: Option<String>,
}
#[derive(Clone, Copy, Debug, Deserialize)]
@@ -521,6 +560,7 @@ fn build_backend(
client_name: &str,
config: &LlmClientConfig,
extra_body: &BTreeMap<String, Value>,
+ reasoning_effort: Option<String>,
) -> RunnerResult<Backend> {
if config.max_retries > MAX_CONFIGURED_RETRIES {
return Err(RunnerError::configuration(format!(
@@ -560,6 +600,7 @@ fn build_backend(
forward_auth: config.forward_auth,
extra_headers: config.extra_headers.clone(),
extra_body: extra_body.clone(),
+ reasoning_effort,
max_retries: config.max_retries,
};
let backend = match config.format {
@@ -949,6 +990,51 @@ new = ["send_message"]
Ok(())
}
+ #[test]
+ fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> {
+ let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
+ let weak = "[targets.weak]\nid = \"weak/model\"\nllm_client = \"anthropic\"";
+ assert!(VALID_CONFIG.contains(strong) && VALID_CONFIG.contains(weak));
+
+ let forced = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \"max\""));
+ runner_from_toml(&forced)?;
+
+ let blank = VALID_CONFIG.replace(strong, &format!("{strong}\nreasoning_effort = \" \""));
+ assert!(error_message(&blank).contains("reasoning_effort must not be empty"));
+
+ let anthropic = VALID_CONFIG.replace(weak, &format!("{weak}\nreasoning_effort = \"high\""));
+ assert!(
+ error_message(&anthropic)
+ .contains("only supported on openai_chat and openai_responses")
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn duplicate_targets_with_conflicting_settings_are_rejected() -> RunnerResult<()> {
+ let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
+ assert!(VALID_CONFIG.contains(strong));
+ // Same model, same client, different effort: the second target could never take effect.
+ let conflicting = VALID_CONFIG.replace(
+ strong,
+ &format!(
+ "{strong}\n\n[targets.strong_max]\nid = \"strong/model\"\nllm_client = \"responses\"\nreasoning_effort = \"max\""
+ ),
+ );
+ assert!(
+ error_message(&conflicting).contains("different reasoning_effort or extra_body"),
+ "{}",
+ error_message(&conflicting)
+ );
+ // An alias with identical settings is still allowed (it only warns).
+ let alias = VALID_CONFIG.replace(
+ strong,
+ &format!("{strong}\n\n[targets.strong_alias]\nid = \"strong/model\"\nllm_client = \"responses\""),
+ );
+ runner_from_toml(&alias)?;
+ Ok(())
+ }
+
#[test]
fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> {
let capability = VALID_CONFIG.replace(
@@ -1348,7 +1434,7 @@ target = "azure"
let Some(client) = config.llm_clients.get("primary") else {
return Err(RunnerError::configuration("primary llm client is missing"));
};
- let backend = build_backend("primary", client, &target.extra_body)?;
+ let backend = build_backend("primary", client, &target.extra_body, None)?;
assert_eq!(
backend.extra_body().get("service_tier"),
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 99fa471da..19adfb1fb 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -513,6 +513,7 @@ fn random_state_with_retries(
forward_auth: false,
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ reasoning_effort: None,
max_retries,
});
let target_models = routes
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 3df685872..92733bd57 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -90,6 +90,7 @@ calls an upstream.
| `llm_client` | Yes | — | Key under `[llm_clients]`. |
| `system_prompt` | No | unset | System prompt prepended when this target serves a completion. |
| `extra_body` | No | `{}` | Values merged into the upstream request when the request does not already set that key. |
+| `reasoning_effort` | No | unset | Reasoning effort forced on every request to this target, replacing the value the caller sent (`reasoning.effort` on `openai_responses`, `reasoning_effort` on `openai_chat`). Rejected on `anthropic_messages` clients. Use it to run one target at a different effort than the client asked for, for example a strong tier at `max` behind a client that sends `high`. Two targets for the same model id on the same `llm_client` collapse into one, so give each effort tier its own `llm_clients` entry (same endpoint, different name). |
Each selected or fallback target is prepared from the routed request independently. A prompt
configured for one target is therefore not carried into another target's fallback request.