Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(llm-client): add request defaults and streaming usage (#184)

nachiketb-nvidia1f5d173128f3
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index 0465dad1e..9e1a4ef81 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -22,13 +22,14 @@ provider SDK.
when set, otherwise the model's default backend.
- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or
`Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,
- static `extra_headers`). The variant fixes the URL path and auth scheme
- (Bearer vs `x-api-key` + `anthropic-version`).
+ static `extra_headers`, default `extra_body` fields). The variant fixes the URL
+ path and auth scheme (Bearer vs `x-api-key` + `anthropic-version`).
- **Model rewrite.** The resolved model name is both the map key and the model id
sent upstream — it overwrites whatever `model` the request arrived with.
- **Streaming is chosen by the request.** If the encoded body has `stream: true`
(i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise
- `LlmResponse::Agg`.
+ `LlmResponse::Agg`. OpenAI Chat streaming requests default
+ `stream_options.include_usage` to `true`; an explicit caller value is preserved.
## Add the dependency
@@ -56,6 +57,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
base_url: "https://api.openai.com/v1".to_string(),
api_key: std::env::var("OPENAI_API_KEY").ok(),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
};
let models = [ModelConfig::new(
@@ -169,6 +171,8 @@ fn build_multi_format_client(
`authorization` / `x-api-key` / `anthropic-version` / `content-type`. So a
caller's placeholder credential never overrides the backend's real key.
- Per-backend static headers go in `HttpBackendConfig::extra_headers`.
+- Per-target top-level request defaults go in `HttpBackendConfig::extra_body`.
+ The merge is shallow and fields already present in the request take precedence.
## Errors
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index f847249e2..2c0555aac 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -6,6 +6,7 @@
use std::{collections::BTreeMap, fmt};
use reqwest::RequestBuilder;
+use serde_json::Value;
use switchyard_protocol::WireFormat;
use crate::error::is_overflow_body;
@@ -39,6 +40,8 @@ pub struct HttpBackendConfig {
pub api_key: Option<String>,
/// Static headers added to every outbound call to this backend.
pub extra_headers: BTreeMap<String, String>,
+ /// Default top-level request fields, applied only when the request omits the key.
+ pub extra_body: BTreeMap<String, Value>,
}
impl fmt::Debug for HttpBackendConfig {
@@ -47,6 +50,7 @@ impl fmt::Debug for HttpBackendConfig {
.field("base_url", &self.base_url)
.field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
.field("extra_headers", &self.extra_headers)
+ .field("extra_body_keys", &self.extra_body.keys())
.finish()
}
}
@@ -124,6 +128,11 @@ impl Backend {
&self.config().extra_headers
}
+ /// Default top-level fields to merge into outbound request bodies.
+ pub fn extra_body(&self) -> &BTreeMap<String, Value> {
+ &self.config().extra_body
+ }
+
/// Whether this backend speaks the Anthropic Messages wire format — the only
/// one with a `count_tokens` endpoint.
pub fn is_anthropic(&self) -> bool {
@@ -186,6 +195,7 @@ mod tests {
base_url: base_url.to_string(),
api_key: Some("secret".to_string()),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
}
}
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 058b66fd4..eeba631fb 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -9,7 +9,7 @@ use std::collections::{BTreeMap, HashMap};
use async_trait::async_trait;
use futures_util::StreamExt;
use reqwest::RequestBuilder;
-use serde_json::Value;
+use serde_json::{Map, Value};
use switchyard_protocol::{
Context, Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
};
@@ -162,6 +162,10 @@ impl TranslatingLlmClient {
// which keeps the caller's original `model`; force the resolved model so
// the upstream always sees the target id.
set_json_model(&mut body, model);
+ merge_extra_body(&mut body, backend.extra_body());
+ if matches!(backend, Backend::OpenAiChat(_)) {
+ ensure_openai_stream_usage(&mut body);
+ }
let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
let builder = self.client.post(url).json(&body);
@@ -454,6 +458,39 @@ fn set_json_model(body: &mut Value, model: &str) {
}
}
+// Applies target defaults without overriding fields supplied by the caller.
+fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {
+ let Value::Object(object) = body else {
+ return;
+ };
+ for (key, value) in extra_body {
+ object.entry(key.clone()).or_insert_with(|| value.clone());
+ }
+}
+
+// Requests streamed Chat usage by default while preserving an explicit caller choice.
+fn ensure_openai_stream_usage(body: &mut Value) {
+ let Value::Object(object) = body else {
+ return;
+ };
+ if object.get("stream").and_then(Value::as_bool) != Some(true) {
+ return;
+ }
+
+ match object.get_mut("stream_options") {
+ Some(Value::Object(options)) => {
+ options
+ .entry("include_usage".to_string())
+ .or_insert(Value::Bool(true));
+ }
+ _ => {
+ let mut options = Map::new();
+ options.insert("include_usage".to_string(), Value::Bool(true));
+ object.insert("stream_options".to_string(), Value::Object(options));
+ }
+ }
+}
+
// Case-insensitive membership test against RESERVED_HEADERS.
fn is_reserved_header(name: &str) -> bool {
RESERVED_HEADERS
@@ -481,6 +518,7 @@ mod tests {
base_url: base_url.to_string(),
api_key: Some("secret".to_string()),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
}
}
@@ -493,6 +531,15 @@ mod tests {
)]
}
+ fn chat_map_with_extra_body(
+ base_url: &str,
+ extra_body: BTreeMap<String, Value>,
+ ) -> Vec<ModelConfig> {
+ let mut backend = config(base_url);
+ backend.extra_body = extra_body;
+ vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
+ }
+
fn truncated_response_server(
content_type: &str,
body: &str,
@@ -771,15 +818,70 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn extra_body_adds_defaults_without_overriding_the_request(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "model": "gpt",
+ "max_tokens": 7,
+ "service_tier": "priority"
+ })))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop"
+ }],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+
+ let extra_body = BTreeMap::from([
+ ("max_tokens".to_string(), json!(999)),
+ ("service_tier".to_string(), json!("priority")),
+ ]);
+ let client = TranslatingLlmClient::new(&chat_map_with_extra_body(
+ &format!("{}/v1", server.uri()),
+ extra_body,
+ ))?;
+ let raw = json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_tokens": 7
+ });
+
+ client
+ .call_rewrite_model_raw(
+ Context::default(),
+ raw,
+ None,
+ Some("gpt"),
+ WireFormat::OpenAiChat,
+ )
+ .await?;
+ Ok(())
+ }
+
#[tokio::test]
async fn streaming_openai_chat_aggregates(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let server = MockServer::start().await;
let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
+ data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n\
data: [DONE]\n\n";
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "stream": true,
+ "stream_options": {"include_usage": true}
+ })))
.respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
.mount(&server)
.await;
@@ -792,6 +894,46 @@ mod tests {
assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
let agg = response.llm_response.into_agg().await?;
assert_eq!(completion_text(&agg), "Hello world");
+ assert_eq!(agg.usage.input_tokens, Some(1));
+ assert_eq!(agg.usage.output_tokens, Some(2));
+ assert_eq!(agg.usage.total_tokens, Some(3));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn streaming_openai_chat_preserves_usage_opt_out(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(wiremock::matchers::body_partial_json(json!({
+ "stream": true,
+ "stream_options": {"include_usage": false}
+ })))
+ .respond_with(
+ ResponseTemplate::new(200).set_body_raw("data: [DONE]\n\n", "text/event-stream"),
+ )
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let raw = json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": true,
+ "stream_options": {"include_usage": false}
+ });
+
+ let response = client
+ .call_rewrite_model_raw(
+ Context::default(),
+ raw,
+ None,
+ Some("gpt"),
+ WireFormat::OpenAiChat,
+ )
+ .await?;
+ assert!(matches!(response, RawResponse::Stream(_)));
Ok(())
}
diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rs
index 7a543c3ca..14376fde4 100644
--- a/crates/libsy/examples/ensemble.rs
+++ b/crates/libsy/examples/ensemble.rs
@@ -411,6 +411,7 @@ async fn main() -> Result<()> {
.map_err(|error| LibsyError::external("reading NVIDIA_API_KEY", error))?,
),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
};
let models: Vec<_> = CANDIDATE_MODELS
.iter()
diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md
index bc9e955c4..ff0ecbba8 100644
--- a/crates/switchyard-server/CONFIGURATION.md
+++ b/crates/switchyard-server/CONFIGURATION.md
@@ -13,8 +13,12 @@ api_key_env = "PROVIDER_API_KEY"
[targets.model]
id = "provider/model"
llm_client = "provider"
+extra_body = { chat_template_kwargs = { enable_thinking = false } }
```
+`extra_body` is target-specific. It shallow-merges top-level provider options into
+the outbound request, while explicit request fields win on conflicts.
+
To support another wire format, add its `ClientFormat` variant and explicit construction match in
`src/config.rs`. Add a client type only when a second implementation exists.
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index b88424bdd..476de372c 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -16,6 +16,7 @@ api_key_env = "API_KEY"
[targets.model_a]
id = "model/a"
llm_client = "example"
+extra_body = { service_tier = "priority" }
[targets.model_b]
id = "model/b"
@@ -60,6 +61,8 @@ Each target references an entry under `llm_clients`. All configured clients use
`anthropic_messages`. Supported algorithms are `noop`, `random`, `passthrough`, and
`llm_classifier`. An `api_key_env` value names an environment variable; the TOML
never contains the secret itself. If omitted, the client sends no authentication.
+Target-level `extra_body` values are shallow-merged into the upstream request when
+the request does not already contain that key.
Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them
for equal weighting. The optional `seed` reproduces the selection sequence for the same call order.
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 5074b4ff5..b1524582d 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -11,6 +11,7 @@ use std::sync::Arc;
use libsy::algorithms::{LlmTaskClassifier, Noop, Passthrough, Random, TaskClassifierConfig};
use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
use serde::Deserialize;
+use serde_json::Value;
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use crate::{ServerError, ServerResult, ServerState};
@@ -94,7 +95,7 @@ impl ServerConfig {
.ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;
model_configs.push(ModelConfig::new(
&target.id,
- build_backend(&target.llm_client, client_config)?,
+ build_backend(&target.llm_client, client_config, &target.extra_body)?,
None,
));
}
@@ -147,6 +148,8 @@ struct LlmClientConfig {
struct TargetConfig {
id: String,
llm_client: String,
+ #[serde(default)]
+ extra_body: BTreeMap<String, Value>,
}
#[derive(Clone, Copy, Debug, Deserialize)]
@@ -196,7 +199,11 @@ impl RouteConfig {
}
}
-fn build_backend(client_name: &str, config: &LlmClientConfig) -> ServerResult<Backend> {
+fn build_backend(
+ client_name: &str,
+ config: &LlmClientConfig,
+ extra_body: &BTreeMap<String, Value>,
+) -> ServerResult<Backend> {
let base_url = config.base_url.trim();
if base_url.is_empty() {
return Err(ServerError::new(format!(
@@ -229,6 +236,7 @@ fn build_backend(client_name: &str, config: &LlmClientConfig) -> ServerResult<Ba
base_url: base_url.to_string(),
api_key,
extra_headers: config.extra_headers.clone(),
+ extra_body: extra_body.clone(),
};
Ok(match config.format {
ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
@@ -317,6 +325,7 @@ fn validate_value(label: &str, value: &str) -> ServerResult<()> {
#[cfg(test)]
mod tests {
use super::*;
+ use serde_json::json;
const VALID_CONFIG: &str = r#"
schema_version = 1
@@ -485,6 +494,39 @@ target = "weak"
Ok(())
}
+ #[test]
+ fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> {
+ let configured = VALID_CONFIG.replacen(
+ "llm_client = \"primary\"",
+ "llm_client = \"primary\"\n\
+ extra_body = { service_tier = \"priority\", \
+ chat_template_kwargs = { enable_thinking = false } }",
+ 1,
+ );
+ let config: ServerConfig = toml::from_str(&configured)
+ .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?;
+ let Some(target) = config.targets.get("classifier") else {
+ return Err(ServerError::new("classifier target is missing"));
+ };
+ let Some(client) = config.llm_clients.get("primary") else {
+ return Err(ServerError::new("primary llm client is missing"));
+ };
+ let backend = build_backend("primary", client, &target.extra_body)?;
+
+ assert_eq!(
+ backend.extra_body().get("service_tier"),
+ Some(&json!("priority"))
+ );
+ assert_eq!(
+ backend
+ .extra_body()
+ .get("chat_template_kwargs")
+ .and_then(|value| value.get("enable_thinking")),
+ Some(&json!(false))
+ );
+ Ok(())
+ }
+
#[test]
fn api_key_environment_reference_is_validated() {
let missing = VALID_CONFIG.replacen(
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index c1c09201b..9735b3cff 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -152,6 +152,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
base_url: base_url.to_string(),
api_key: Some("test-key".to_string()),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
});
let target_models = routes
.iter()
@@ -569,6 +570,7 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<S
base_url: url.to_string(),
api_key: Some("k".to_string()),
extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
};
let strong: Arc<dyn RoutedLlmClient> =
Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(