Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(llm-client): add request defaults and streaming usage (#184)
nachiketb-nvidia1f5d173128f3
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdindex 0465dad1e..9e1a4ef81 100644--- a/crates/libsy-llm-client/README.md+++ b/crates/libsy-llm-client/README.md@@ -22,13 +22,14 @@ provider SDK.when set, otherwise the model's default backend.- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or`Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,- static `extra_headers`). The variant fixes the URL path and auth scheme- (Bearer vs `x-api-key` + `anthropic-version`).+ static `extra_headers`, default `extra_body` fields). The variant fixes the URL+ path and auth scheme (Bearer vs `x-api-key` + `anthropic-version`).- **Model rewrite.** The resolved model name is both the map key and the model idsent upstream — it overwrites whatever `model` the request arrived with.- **Streaming is chosen by the request.** If the encoded body has `stream: true`(i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise- `LlmResponse::Agg`.+ `LlmResponse::Agg`. OpenAI Chat streaming requests default+ `stream_options.include_usage` to `true`; an explicit caller value is preserved.## Add the dependency@@ -56,6 +57,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {base_url: "https://api.openai.com/v1".to_string(),api_key: std::env::var("OPENAI_API_KEY").ok(),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),};let models = [ModelConfig::new(@@ -169,6 +171,8 @@ fn build_multi_format_client(`authorization` / `x-api-key` / `anthropic-version` / `content-type`. So acaller's placeholder credential never overrides the backend's real key.- Per-backend static headers go in `HttpBackendConfig::extra_headers`.+- Per-target top-level request defaults go in `HttpBackendConfig::extra_body`.+ The merge is shallow and fields already present in the request take precedence.## Errorsdiff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex f847249e2..2c0555aac 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -6,6 +6,7 @@use std::{collections::BTreeMap, fmt};use reqwest::RequestBuilder;+use serde_json::Value;use switchyard_protocol::WireFormat;use crate::error::is_overflow_body;@@ -39,6 +40,8 @@ pub struct HttpBackendConfig {pub api_key: Option<String>,/// Static headers added to every outbound call to this backend.pub extra_headers: BTreeMap<String, String>,+ /// Default top-level request fields, applied only when the request omits the key.+ pub extra_body: BTreeMap<String, Value>,}impl fmt::Debug for HttpBackendConfig {@@ -47,6 +50,7 @@ impl fmt::Debug for HttpBackendConfig {.field("base_url", &self.base_url).field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]")).field("extra_headers", &self.extra_headers)+ .field("extra_body_keys", &self.extra_body.keys()).finish()}}@@ -124,6 +128,11 @@ impl Backend {&self.config().extra_headers}+ /// Default top-level fields to merge into outbound request bodies.+ pub fn extra_body(&self) -> &BTreeMap<String, Value> {+ &self.config().extra_body+ }+/// Whether this backend speaks the Anthropic Messages wire format — the only/// one with a `count_tokens` endpoint.pub fn is_anthropic(&self) -> bool {@@ -186,6 +195,7 @@ mod tests {base_url: base_url.to_string(),api_key: Some("secret".to_string()),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),}}diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 058b66fd4..eeba631fb 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -9,7 +9,7 @@ use std::collections::{BTreeMap, HashMap};use async_trait::async_trait;use futures_util::StreamExt;use reqwest::RequestBuilder;-use serde_json::Value;+use serde_json::{Map, Value};use switchyard_protocol::{Context, Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient,};@@ -162,6 +162,10 @@ impl TranslatingLlmClient {// which keeps the caller's original `model`; force the resolved model so// the upstream always sees the target id.set_json_model(&mut body, model);+ merge_extra_body(&mut body, backend.extra_body());+ if matches!(backend, Backend::OpenAiChat(_)) {+ ensure_openai_stream_usage(&mut body);+ }let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);let builder = self.client.post(url).json(&body);@@ -454,6 +458,39 @@ fn set_json_model(body: &mut Value, model: &str) {}}+// Applies target defaults without overriding fields supplied by the caller.+fn merge_extra_body(body: &mut Value, extra_body: &BTreeMap<String, Value>) {+ let Value::Object(object) = body else {+ return;+ };+ for (key, value) in extra_body {+ object.entry(key.clone()).or_insert_with(|| value.clone());+ }+}++// Requests streamed Chat usage by default while preserving an explicit caller choice.+fn ensure_openai_stream_usage(body: &mut Value) {+ let Value::Object(object) = body else {+ return;+ };+ if object.get("stream").and_then(Value::as_bool) != Some(true) {+ return;+ }++ match object.get_mut("stream_options") {+ Some(Value::Object(options)) => {+ options+ .entry("include_usage".to_string())+ .or_insert(Value::Bool(true));+ }+ _ => {+ let mut options = Map::new();+ options.insert("include_usage".to_string(), Value::Bool(true));+ object.insert("stream_options".to_string(), Value::Object(options));+ }+ }+}+// Case-insensitive membership test against RESERVED_HEADERS.fn is_reserved_header(name: &str) -> bool {RESERVED_HEADERS@@ -481,6 +518,7 @@ mod tests {base_url: base_url.to_string(),api_key: Some("secret".to_string()),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),}}@@ -493,6 +531,15 @@ mod tests {)]}+ fn chat_map_with_extra_body(+ base_url: &str,+ extra_body: BTreeMap<String, Value>,+ ) -> Vec<ModelConfig> {+ let mut backend = config(base_url);+ backend.extra_body = extra_body;+ vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]+ }+fn truncated_response_server(content_type: &str,body: &str,@@ -771,15 +818,70 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn extra_body_adds_defaults_without_overriding_the_request(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(wiremock::matchers::body_partial_json(json!({+ "model": "gpt",+ "max_tokens": 7,+ "service_tier": "priority"+ })))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop"+ }],+ "usage": {}+ })))+ .mount(&server)+ .await;++ let extra_body = BTreeMap::from([+ ("max_tokens".to_string(), json!(999)),+ ("service_tier".to_string(), json!("priority")),+ ]);+ let client = TranslatingLlmClient::new(&chat_map_with_extra_body(+ &format!("{}/v1", server.uri()),+ extra_body,+ ))?;+ let raw = json!({+ "model": "client-facing",+ "messages": [{"role": "user", "content": "hi"}],+ "max_tokens": 7+ });++ client+ .call_rewrite_model_raw(+ Context::default(),+ raw,+ None,+ Some("gpt"),+ WireFormat::OpenAiChat,+ )+ .await?;+ Ok(())+ }+#[tokio::test]async fn streaming_openai_chat_aggregates() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {let server = MockServer::start().await;let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\+ data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":2,\"total_tokens\":3}}\n\n\data: [DONE]\n\n";Mock::given(method("POST")).and(path("/v1/chat/completions"))+ .and(wiremock::matchers::body_partial_json(json!({+ "stream": true,+ "stream_options": {"include_usage": true}+ }))).respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")).mount(&server).await;@@ -792,6 +894,46 @@ mod tests {assert!(matches!(response.llm_response, LlmResponse::Stream(_)));let agg = response.llm_response.into_agg().await?;assert_eq!(completion_text(&agg), "Hello world");+ assert_eq!(agg.usage.input_tokens, Some(1));+ assert_eq!(agg.usage.output_tokens, Some(2));+ assert_eq!(agg.usage.total_tokens, Some(3));+ Ok(())+ }++ #[tokio::test]+ async fn streaming_openai_chat_preserves_usage_opt_out(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(wiremock::matchers::body_partial_json(json!({+ "stream": true,+ "stream_options": {"include_usage": false}+ })))+ .respond_with(+ ResponseTemplate::new(200).set_body_raw("data: [DONE]\n\n", "text/event-stream"),+ )+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let raw = json!({+ "model": "client-facing",+ "messages": [{"role": "user", "content": "hi"}],+ "stream": true,+ "stream_options": {"include_usage": false}+ });++ let response = client+ .call_rewrite_model_raw(+ Context::default(),+ raw,+ None,+ Some("gpt"),+ WireFormat::OpenAiChat,+ )+ .await?;+ assert!(matches!(response, RawResponse::Stream(_)));Ok(())}diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rsindex 7a543c3ca..14376fde4 100644--- a/crates/libsy/examples/ensemble.rs+++ b/crates/libsy/examples/ensemble.rs@@ -411,6 +411,7 @@ async fn main() -> Result<()> {.map_err(|error| LibsyError::external("reading NVIDIA_API_KEY", error))?,),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),};let models: Vec<_> = CANDIDATE_MODELS.iter()diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.mdindex bc9e955c4..ff0ecbba8 100644--- a/crates/switchyard-server/CONFIGURATION.md+++ b/crates/switchyard-server/CONFIGURATION.md@@ -13,8 +13,12 @@ api_key_env = "PROVIDER_API_KEY"[targets.model]id = "provider/model"llm_client = "provider"+extra_body = { chat_template_kwargs = { enable_thinking = false } }```+`extra_body` is target-specific. It shallow-merges top-level provider options into+the outbound request, while explicit request fields win on conflicts.+To support another wire format, add its `ClientFormat` variant and explicit construction match in`src/config.rs`. Add a client type only when a second implementation exists.diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex b88424bdd..476de372c 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -16,6 +16,7 @@ api_key_env = "API_KEY"[targets.model_a]id = "model/a"llm_client = "example"+extra_body = { service_tier = "priority" }[targets.model_b]id = "model/b"@@ -60,6 +61,8 @@ Each target references an entry under `llm_clients`. All configured clients use`anthropic_messages`. Supported algorithms are `noop`, `random`, `passthrough`, and`llm_classifier`. An `api_key_env` value names an environment variable; the TOMLnever contains the secret itself. If omitted, the client sends no authentication.+Target-level `extra_body` values are shallow-merged into the upstream request when+the request does not already contain that key.Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit themfor equal weighting. The optional `seed` reproduces the selection sequence for the same call order.diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 5074b4ff5..b1524582d 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -11,6 +11,7 @@ use std::sync::Arc;use libsy::algorithms::{LlmTaskClassifier, Noop, Passthrough, Random, TaskClassifierConfig};use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};use serde::Deserialize;+use serde_json::Value;use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};use crate::{ServerError, ServerResult, ServerState};@@ -94,7 +95,7 @@ impl ServerConfig {.ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;model_configs.push(ModelConfig::new(&target.id,- build_backend(&target.llm_client, client_config)?,+ build_backend(&target.llm_client, client_config, &target.extra_body)?,None,));}@@ -147,6 +148,8 @@ struct LlmClientConfig {struct TargetConfig {id: String,llm_client: String,+ #[serde(default)]+ extra_body: BTreeMap<String, Value>,}#[derive(Clone, Copy, Debug, Deserialize)]@@ -196,7 +199,11 @@ impl RouteConfig {}}-fn build_backend(client_name: &str, config: &LlmClientConfig) -> ServerResult<Backend> {+fn build_backend(+ client_name: &str,+ config: &LlmClientConfig,+ extra_body: &BTreeMap<String, Value>,+) -> ServerResult<Backend> {let base_url = config.base_url.trim();if base_url.is_empty() {return Err(ServerError::new(format!(@@ -229,6 +236,7 @@ fn build_backend(client_name: &str, config: &LlmClientConfig) -> ServerResult<Babase_url: base_url.to_string(),api_key,extra_headers: config.extra_headers.clone(),+ extra_body: extra_body.clone(),};Ok(match config.format {ClientFormat::OpenAiChat => Backend::OpenAiChat(http),@@ -317,6 +325,7 @@ fn validate_value(label: &str, value: &str) -> ServerResult<()> {#[cfg(test)]mod tests {use super::*;+ use serde_json::json;const VALID_CONFIG: &str = r#"schema_version = 1@@ -485,6 +494,39 @@ target = "weak"Ok(())}+ #[test]+ fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> {+ let configured = VALID_CONFIG.replacen(+ "llm_client = \"primary\"",+ "llm_client = \"primary\"\n\+ extra_body = { service_tier = \"priority\", \+ chat_template_kwargs = { enable_thinking = false } }",+ 1,+ );+ let config: ServerConfig = toml::from_str(&configured)+ .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?;+ let Some(target) = config.targets.get("classifier") else {+ return Err(ServerError::new("classifier target is missing"));+ };+ let Some(client) = config.llm_clients.get("primary") else {+ return Err(ServerError::new("primary llm client is missing"));+ };+ let backend = build_backend("primary", client, &target.extra_body)?;++ assert_eq!(+ backend.extra_body().get("service_tier"),+ Some(&json!("priority"))+ );+ assert_eq!(+ backend+ .extra_body()+ .get("chat_template_kwargs")+ .and_then(|value| value.get("enable_thinking")),+ Some(&json!(false))+ );+ Ok(())+ }+#[test]fn api_key_environment_reference_is_validated() {let missing = VALID_CONFIG.replacen(diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex c1c09201b..9735b3cff 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -152,6 +152,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Serverbase_url: base_url.to_string(),api_key: Some("test-key".to_string()),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),});let target_models = routes.iter()@@ -569,6 +570,7 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<Sbase_url: url.to_string(),api_key: Some("k".to_string()),extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),};let strong: Arc<dyn RoutedLlmClient> =Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(