Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(switchyard-llm-client): add HTTP LLM client crate (#79)

Greg Clark0bc032adb27f
diff --git a/Cargo.lock b/Cargo.lock
index b432a6d95..9662c3551 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 4
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -67,6 +76,16 @@ dependencies = [
"rustversion",
]
+[[package]]
+name = "assert-json-diff"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -348,6 +367,24 @@ dependencies = [
"libc",
]
+[[package]]
+name = "deadpool"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b"
+dependencies = [
+ "deadpool-runtime",
+ "lazy_static",
+ "num_cpus",
+ "tokio",
+]
+
+[[package]]
+name = "deadpool-runtime"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
+
[[package]]
name = "diff"
version = "0.1.13"
@@ -593,6 +630,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
[[package]]
name = "http"
version = "1.4.2"
@@ -851,6 +894,12 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
[[package]]
name = "libc"
version = "0.2.186"
@@ -940,6 +989,16 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -1260,6 +1319,35 @@ dependencies = [
"bitflags",
]
+[[package]]
+name = "regex"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
[[package]]
name = "reqwest"
version = "0.12.28"
@@ -1629,6 +1717,22 @@ dependencies = [
"tokio-stream",
]
+[[package]]
+name = "switchyard-llm-client"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "futures",
+ "futures-util",
+ "reqwest",
+ "serde_json",
+ "switchyard-protocol",
+ "switchyard-translation",
+ "thiserror",
+ "tokio",
+ "wiremock",
+]
+
[[package]]
name = "switchyard-protocol"
version = "0.1.0"
@@ -2199,6 +2303,29 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "wiremock"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"
+dependencies = [
+ "assert-json-diff",
+ "base64",
+ "deadpool",
+ "futures",
+ "http",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "log",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_json",
+ "tokio",
+ "url",
+]
+
[[package]]
name = "writeable"
version = "0.6.3"
diff --git a/Cargo.toml b/Cargo.toml
index ec2ee2c7a..551d52cfa 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,7 @@ resolver = "2"
members = [
"crates/libsy",
"crates/libsy-examples",
+ "crates/libsy-llm-client",
"crates/switchyard-components",
"crates/switchyard-components-v2",
"crates/switchyard-components-v2-macros",
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml
new file mode 100644
index 000000000..67c338931
--- /dev/null
+++ b/crates/libsy-llm-client/Cargo.toml
@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "switchyard-llm-client"
+version = "0.1.0"
+description = "HTTP LLM client speaking Switchyard's neutral IR directly"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+switchyard-protocol = { path = "../protocol", version = "0.1.0" }
+switchyard-translation = { path = "../switchyard-translation", version = "0.1.0" }
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
+async-trait = "0.1"
+futures-util = "0.3"
+serde_json = "1"
+thiserror = "2"
+
+[dev-dependencies]
+futures = "0.3"
+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+wiremock = "0.6"
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
new file mode 100644
index 000000000..e769123f6
--- /dev/null
+++ b/crates/libsy-llm-client/README.md
@@ -0,0 +1,193 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# libsy-llm-client
+
+An HTTP client that speaks Switchyard's neutral IR directly. You hand it a
+[`switchyard_protocol::Request`] and a model name; it looks up the configured backend,
+encodes the request to that backend's wire format, adds auth and forwards your
+headers, makes the call with a shared `reqwest::Client`, and decodes the reply
+back into a [`switchyard_protocol::Response`] — buffered or streamed.
+
+It depends only on `switchyard-protocol` and `switchyard-translation`; no server, no
+provider SDK.
+
+## Concepts
+
+- **Model configs.** A client is built from [`ModelConfig`] values. Each model has a
+ default [`Backend`] and can have additional backends for other wire formats.
+ [`TranslatingLlmClient::call_rewrite_model`] uses the request's metadata wire format
+ when set, otherwise the model's default backend.
+- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or
+ `Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,
+ static `extra_headers`). The variant fixes the URL path and auth scheme
+ (Bearer vs `x-api-key` + `anthropic-version`).
+- **Model rewrite.** The resolved model name is both the map key and the model id
+ sent upstream — it overwrites whatever `model` the request arrived with.
+- **Streaming is chosen by the request.** If the encoded body has `stream: true`
+ (i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise
+ `LlmResponse::Agg`.
+
+## Add the dependency
+
+Within this workspace:
+
+```toml
+[dependencies]
+switchyard-llm-client = { path = "../libsy-llm-client" }
+switchyard-protocol = { path = "../libsy-protocol" }
+switchyard-translation = { path = "../switchyard-translation" } # for WireFormat
+```
+
+## Quickstart
+
+### Build a client
+
+```rust
+use std::collections::BTreeMap;
+use switchyard_llm_client::{
+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
+};
+
+fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
+ let openai = HttpBackendConfig {
+ base_url: "https://api.openai.com/v1".to_string(),
+ api_key: std::env::var("OPENAI_API_KEY").ok(),
+ extra_headers: BTreeMap::new(),
+ };
+
+ let models = [ModelConfig::new(
+ "gpt-4o-mini",
+ Backend::OpenAiChat(openai),
+ None,
+ )];
+
+ TranslatingLlmClient::new(&models)
+}
+```
+
+### Buffered call
+
+```rust
+use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};
+use switchyard_protocol::{completion_text, text_request, Context, LlmResponse, Request};
+
+async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<String> {
+ let request = Request {
+ llm_request: text_request(None, "Say hello in five words."),
+ raw_request: None,
+ metadata: None,
+ };
+
+ // model_name wins over request.llm_request.model; it is also sent upstream.
+ let response = client
+ .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))
+ .await?;
+
+ match response.llm_response {
+ LlmResponse::Agg(agg) => Ok(completion_text(&agg)),
+ LlmResponse::Stream(_) => Err(LlmClientError::Stream(
+ "expected a buffered response".to_string(),
+ )),
+ }
+}
+```
+
+### Streaming call
+
+Set `stream` on the IR request and drive the returned chunk stream:
+
+```rust
+use futures_util::StreamExt;
+use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};
+use switchyard_protocol::{text_request, Context, LlmResponse, LlmResponseChunk, Request};
+
+async fn stream(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<()> {
+ let mut llm_request = text_request(None, "Count to five.");
+ llm_request.stream = true;
+ let request = Request { llm_request, raw_request: None, metadata: None };
+
+ let response = client
+ .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))
+ .await?;
+
+ if let LlmResponse::Stream(mut chunks) = response.llm_response {
+ while let Some(item) = chunks.next().await {
+ match item {
+ Ok(LlmResponseChunk::TextDelta { text, .. }) => print!("{text}"),
+ Ok(_) => {} // usage, tool-call deltas, message start/stop
+ Err(error) => return Err(LlmClientError::Stream(error.to_string())),
+ }
+ }
+ }
+ Ok(())
+}
+```
+
+## Cross-format translation
+
+The request/response are translated through the neutral IR, so the inbound shape
+you build and the backend's wire format are independent. Pointing a
+`WireFormat::AnthropicMessages` backend at an Anthropic endpoint while building
+requests with the same helpers works the same way. Set `request.metadata.wire_format`
+to select a non-default backend before calling `call_rewrite_model`. Register several
+formats under one model to serve it over more than one upstream API:
+
+```rust
+use switchyard_llm_client::{
+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient,
+};
+
+fn build_multi_format_client(
+ openai_chat: HttpBackendConfig,
+ openai_responses: HttpBackendConfig,
+ anthropic: HttpBackendConfig,
+) -> switchyard_llm_client::Result<TranslatingLlmClient> {
+ let models = [ModelConfig::new(
+ "my-model",
+ Backend::OpenAiChat(openai_chat),
+ Some(vec![
+ Backend::OpenAiResponses(openai_responses),
+ Backend::Anthropic(anthropic),
+ ]),
+ )];
+
+ TranslatingLlmClient::new(&models)
+}
+```
+
+## Headers & auth
+
+- The `Backend` variant sets auth: OpenAI formats send `Authorization: Bearer <key>`;
+ Anthropic sends `x-api-key: <key>` plus `anthropic-version`.
+- `request.metadata.http_headers` are forwarded upstream, **except** reserved ones:
+ `host`, `content-length`, `connection`, and the backend-owned
+ `authorization` / `x-api-key` / `anthropic-version` / `content-type`. So a
+ caller's placeholder credential never overrides the backend's real key.
+- Per-backend static headers go in `HttpBackendConfig::extra_headers`.
+
+## Errors
+
+`call_rewrite_model` returns [`LlmClientError`]:
+
+| Variant | When |
+|---------|------|
+| `MissingModel` | no `model_name` arg and no `request.llm_request.model` |
+| `UnknownModel(model)` | model has no backends configured |
+| `UnknownModelFormat { model, format }` | model has no backend for that format |
+| `Translation(msg)` | encode/decode failed in the translation engine |
+| `Transport(msg)` | connect/timeout/transport failure |
+| `ContextWindowExceeded { model, message }` | upstream 400 detected as a context overflow (checked before `UpstreamHttp`, so callers can evict-and-retry) |
+| `UpstreamHttp { status, body }` | any other non-2xx upstream response |
+| `Stream(msg)` | mid-stream read / malformed frame |
+
+[`switchyard_protocol::Request`]: ../libsy-protocol
+[`switchyard_protocol::Response`]: ../libsy-protocol
+[`libsy-proxy`]: ../libsy-proxy
+[`Backend`]: src/backend.rs
+[`HttpBackendConfig`]: src/backend.rs
+[`ModelConfig`]: src/client.rs
+[`TranslatingLlmClient::call_rewrite_model`]: src/client.rs
+[`LlmClientError`]: src/error.rs
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
new file mode 100644
index 000000000..c907f290d
--- /dev/null
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -0,0 +1,256 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Per-provider backend configuration: wire format, upstream URL, and auth.
+
+use std::{collections::BTreeMap, fmt};
+
+use reqwest::RequestBuilder;
+use switchyard_protocol::WireFormat;
+
+use crate::error::is_overflow_body;
+
+const ANTHROPIC_VERSION: &str = "2023-06-01";
+
+// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
+// provider-wrap is a one-line entry here, not a fork of the parsing logic.
+const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
+ "maximum context length",
+ "context length exceeded",
+ "context window",
+ "context length is only",
+ "please reduce the length of the input",
+];
+
+// Anthropic has no structured `error.code`, so detection is phrase-based only.
+const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
+ "prompt is too long",
+ "maximum number of tokens",
+ "context window",
+ "context length",
+];
+
+/// Shared HTTP configuration for one upstream backend.
+#[derive(Clone)]
+pub struct HttpBackendConfig {
+ /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
+ pub base_url: String,
+ /// API key for the provider, loaded by the caller. `None` sends no auth.
+ pub api_key: Option<String>,
+ /// Static headers added to every outbound call to this backend.
+ pub extra_headers: BTreeMap<String, String>,
+}
+
+impl fmt::Debug for HttpBackendConfig {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("HttpBackendConfig")
+ .field("base_url", &self.base_url)
+ .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
+ .field("extra_headers", &self.extra_headers)
+ .finish()
+ }
+}
+
+/// A configured upstream backend, one variant per built-in wire format.
+///
+/// The variant fixes the wire format, URL path, and auth scheme together so no
+/// invalid combination can be constructed.
+#[derive(Clone, Debug)]
+pub enum Backend {
+ /// OpenAI-compatible Chat Completions API.
+ OpenAiChat(HttpBackendConfig),
+ /// OpenAI Responses API.
+ OpenAiResponses(HttpBackendConfig),
+ /// Anthropic Messages API.
+ Anthropic(HttpBackendConfig),
+}
+
+impl Backend {
+ /// The wire format the request IR is encoded to for this backend.
+ pub fn wire_format(&self) -> WireFormat {
+ match self {
+ Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
+ Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
+ Backend::Anthropic(_) => WireFormat::AnthropicMessages,
+ }
+ }
+
+ // Shared HTTP config, regardless of variant.
+ fn config(&self) -> &HttpBackendConfig {
+ match self {
+ Backend::OpenAiChat(config)
+ | Backend::OpenAiResponses(config)
+ | Backend::Anthropic(config) => config,
+ }
+ }
+
+ /// The fully resolved upstream URL for this backend's endpoint.
+ ///
+ /// Tolerates base URLs that already include the provider path (or a bare
+ /// `/v1`), matching the join rules of the existing native backends.
+ pub fn url(&self) -> String {
+ let base_url = self.config().base_url.trim_end_matches('/');
+ match self {
+ Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
+ Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
+ Backend::Anthropic(_) => anthropic_url(base_url),
+ }
+ }
+
+ /// Applies this backend's auth and version headers to a request builder.
+ ///
+ /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
+ /// `x-api-key: <key>` plus the required `anthropic-version` header.
+ pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
+ let api_key = self.config().api_key.as_deref();
+ match self {
+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
+ if let Some(api_key) = api_key {
+ builder = builder.bearer_auth(api_key);
+ }
+ }
+ Backend::Anthropic(_) => {
+ builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
+ if let Some(api_key) = api_key {
+ builder = builder.header("x-api-key", api_key);
+ }
+ }
+ }
+ builder
+ }
+
+ /// Static per-backend headers to forward on every call.
+ pub fn extra_headers(&self) -> &BTreeMap<String, String> {
+ &self.config().extra_headers
+ }
+
+ /// Whether an upstream 400 `body` looks like a context-window overflow for
+ /// this backend's provider.
+ pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
+ match self {
+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
+ body,
+ |value| {
+ value
+ .get("error")
+ .and_then(|err| err.get("code"))
+ .and_then(serde_json::Value::as_str)
+ == Some("context_length_exceeded")
+ },
+ OPENAI_OVERFLOW_PHRASES,
+ ),
+ Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
+ }
+ }
+}
+
+// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
+fn openai_url(base_url: &str, suffix: &str) -> String {
+ let base_root = base_url
+ .strip_suffix("/chat/completions")
+ .or_else(|| base_url.strip_suffix("/responses"))
+ .unwrap_or(base_url);
+ format!("{base_root}{suffix}")
+}
+
+// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
+fn anthropic_url(base_url: &str) -> String {
+ if base_url.ends_with("/v1/messages") {
+ base_url.to_string()
+ } else if base_url.ends_with("/v1") {
+ format!("{base_url}/messages")
+ } else {
+ format!("{base_url}/v1/messages")
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn config(base_url: &str) -> HttpBackendConfig {
+ HttpBackendConfig {
+ base_url: base_url.to_string(),
+ api_key: Some("secret".to_string()),
+ extra_headers: BTreeMap::new(),
+ }
+ }
+
+ #[test]
+ fn openai_chat_url_joins_bare_v1() {
+ let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
+ assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
+ }
+
+ #[test]
+ fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
+ assert_eq!(
+ Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
+ "https://api.openai.com/v1/chat/completions"
+ );
+ assert_eq!(
+ Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
+ "https://api.openai.com/v1/chat/completions"
+ );
+ }
+
+ #[test]
+ fn openai_responses_url_uses_responses_path() {
+ assert_eq!(
+ Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
+ "https://api.openai.com/v1/responses"
+ );
+ }
+
+ #[test]
+ fn anthropic_url_join_cases() {
+ assert_eq!(
+ Backend::Anthropic(config("https://api.anthropic.com")).url(),
+ "https://api.anthropic.com/v1/messages"
+ );
+ assert_eq!(
+ Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
+ "https://api.anthropic.com/v1/messages"
+ );
+ assert_eq!(
+ Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
+ "https://api.anthropic.com/v1/messages"
+ );
+ }
+
+ #[test]
+ fn wire_format_matches_variant() {
+ assert_eq!(
+ Backend::OpenAiChat(config("x")).wire_format(),
+ WireFormat::OpenAiChat
+ );
+ assert_eq!(
+ Backend::OpenAiResponses(config("x")).wire_format(),
+ WireFormat::OpenAiResponses
+ );
+ assert_eq!(
+ Backend::Anthropic(config("x")).wire_format(),
+ WireFormat::AnthropicMessages
+ );
+ }
+
+ #[test]
+ fn openai_detects_canonical_and_wrapped_overflow() {
+ let backend = Backend::OpenAiChat(config("x"));
+ assert!(backend
+ .is_context_overflow(r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#));
+ // NVIDIA/LiteLLM message wrap with no structured code.
+ assert!(backend.is_context_overflow(
+ r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
+ ));
+ assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
+ }
+
+ #[test]
+ fn anthropic_detects_prompt_too_long() {
+ let backend = Backend::Anthropic(config("x"));
+ assert!(backend
+ .is_context_overflow(r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#));
+ assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
+ }
+}
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
new file mode 100644
index 000000000..a745ddef5
--- /dev/null
+++ b/crates/libsy-llm-client/src/client.rs
@@ -0,0 +1,828 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral
+//! request, call the configured backend over HTTP, decode the neutral response.
+
+use std::collections::{BTreeMap, HashMap};
+
+use async_trait::async_trait;
+use futures_util::StreamExt;
+use reqwest::RequestBuilder;
+use serde_json::Value;
+use switchyard_protocol::{
+ Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient,
+};
+use switchyard_translation::{
+ decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
+ encode_request, encode_stream, WireFormat,
+};
+
+use crate::backend::Backend;
+use crate::error::{LlmClientError, Result};
+use crate::raw::RawResponse;
+
+// TODO: Why is this here? What does it do?
+// Headers this client owns or that are hop-by-hop; never forwarded from the
+// caller's metadata. Auth/version/content-type are set by the backend or the
+// JSON body, so a forwarded copy would either be ignored or conflict. Compared
+// case-insensitively. Aligns with `_SENSITIVE_HEADERS` in the Python
+// `switchyard/lib/request_metadata.py` forwarding logic.
+const RESERVED_HEADERS: &[&str] = &[
+ "host",
+ "content-length",
+ "connection",
+ "authorization",
+ "proxy-authorization",
+ "proxy-authenticate",
+ "cookie",
+ "set-cookie",
+ "x-api-key",
+ "anthropic-version",
+ "content-type",
+];
+
+/// How one model is served: the `default_backend` used when the request does not
+/// pin a wire format, plus any `other_backends` reachable over additional formats.
+#[derive(Clone, Debug)]
+pub struct ModelConfig {
+ model_name: String,
+ default_backend: Backend,
+ other_backends: Option<Vec<Backend>>,
+}
+
+impl ModelConfig {
+ /// A model named `model_name` served by `default_backend`, optionally reachable
+ /// over additional wire formats via `other_backends`.
+ pub fn new(
+ model_name: impl Into<String>,
+ default_backend: Backend,
+ other_backends: Option<Vec<Backend>>,
+ ) -> Self {
+ Self {
+ model_name: model_name.into(),
+ default_backend,
+ other_backends,
+ }
+ }
+}
+
+/// A client that dispatches neutral-IR requests to per-model HTTP backends.
+///
+/// Construct it with a list of [`ModelConfig`]s — one per model, each naming a
+/// default [`Backend`] and any additional per-format backends. Each call resolves
+/// the model and wire format, encodes the request to that backend's wire format,
+/// applies auth and forwarded headers, sends the HTTP request with a shared
+/// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered
+/// or streamed).
+pub struct TranslatingLlmClient {
+ model_to_config: HashMap<String, ModelConfig>,
+ client: reqwest::Client,
+}
+
+impl TranslatingLlmClient {
+ /// Builds a client over the given [`ModelConfig`]s, with a fresh shared HTTP
+ /// client and the built-in translation codecs.
+ pub fn new(model_configs: &[ModelConfig]) -> Result<Self> {
+ let client = reqwest::Client::builder().build().map_err(|error| {
+ LlmClientError::Transport(format!("failed to build HTTP client: {error}"))
+ })?;
+ let model_to_config = model_configs
+ .iter()
+ .map(|config| (config.model_name.clone(), config.clone()))
+ .collect();
+
+ Ok(Self {
+ model_to_config,
+ client,
+ })
+ }
+
+ /// The backend serving `model` over `format` — the default backend when its
+ /// format matches, otherwise a matching entry in `other_backends`; `None` when
+ /// the model is unknown or has no backend for `format`.
+ pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> {
+ self.model_to_config.get(model).and_then(|config| {
+ if config.default_backend.wire_format() == format {
+ Some(&config.default_backend)
+ } else {
+ config
+ .other_backends
+ .as_ref()
+ .and_then(|backends| backends.iter().find(|b| b.wire_format() == format))
+ }
+ })
+ }
+
+ /// Calls the backend for `model_name` (or the request's own model), over the
+ /// wire format the request pins in its metadata (else the model's default
+ /// backend), and returns the neutral response.
+ ///
+ /// Resolution: `model_name` wins over `request.llm_request.model`; the
+ /// resolved name is both the outer map key and the model id written into the
+ /// request before translation. Errors with [`LlmClientError::MissingModel`]
+ /// when neither is set, [`LlmClientError::UnknownModel`] when the model has
+ /// no backends, and [`LlmClientError::UnknownModelFormat`] when the model has
+ /// no backend for `format`.
+ pub async fn call_rewrite_model(
+ &self,
+ _ctx: Context,
+ request: Request,
+ model_name: Option<&str>,
+ ) -> Result<Response> {
+ // Own the request's parts so the model can be set without a `mut` param
+ // and without cloning the messages. `raw_request` is unused here.
+ let Request {
+ mut llm_request,
+ metadata,
+ ..
+ } = request;
+
+ let model = model_name
+ .map(str::to_string)
+ .or_else(|| llm_request.model.clone())
+ .ok_or(LlmClientError::MissingModel)?;
+
+ let orig_format = metadata.as_ref().and_then(|m| m.wire_format);
+ let wire_format = orig_format.unwrap_or(
+ self.model_to_config
+ .get(&model)
+ .map(|config| config.default_backend.wire_format())
+ .ok_or(LlmClientError::UnknownModel(model.clone()))?,
+ );
+ let backend =
+ self.backend_for(&model, wire_format)
+ .ok_or(LlmClientError::UnknownModelFormat {
+ model: model.clone(),
+ format: wire_format,
+ })?;
+
+ // The resolved name is the upstream model id (per the crate contract).
+ llm_request.model = Some(model.clone());
+
+ let mut body = encode_request(&llm_request, wire_format)
+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;
+ // `encode_request` round-trips a preserved same-format body verbatim,
+ // which keeps the caller's original `model`; force the resolved model so
+ // the upstream always sees the target id.
+ set_json_model(&mut body, &model);
+ let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+
+ let builder = self.client.post(backend.url()).json(&body);
+ let builder = forward_metadata_headers(builder, metadata.as_ref());
+ let builder = apply_extra_headers(builder, backend);
+ let builder = backend.apply_auth(builder);
+
+ let http_response = builder
+ .send()
+ .await
+ .map_err(|error| LlmClientError::Transport(error.to_string()))?;
+ let status = http_response.status();
+ if !status.is_success() {
+ let body = http_response
+ .text()
+ .await
+ .unwrap_or_else(|error| format!("<failed to read error body: {error}>"));
+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
+ return Err(LlmClientError::ContextWindowExceeded {
+ model,
+ message: body,
+ });
+ }
+ return Err(LlmClientError::UpstreamHttp {
+ status: status.as_u16(),
+ body,
+ });
+ }
+
+ let llm_response = if streaming {
+ // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
+ // transport-agnostic and lives in `switchyard-translation`.
+ let bytes = http_response.bytes_stream().map(|chunk| {
+ chunk.map(|bytes| bytes.to_vec()).map_err(|error| {
+ Box::new(LlmClientError::Stream(format!(
+ "stream read failed: {error}"
+ ))) as Box<dyn std::error::Error + Send + Sync>
+ })
+ });
+ let chunks = decode_stream(bytes, wire_format)
+ .map_err(|error| LlmClientError::Stream(error.to_string()))?;
+ LlmResponse::Stream(chunks)
+ } else {
+ let body = http_response.json::<Value>().await.map_err(|error| {
+ LlmClientError::Transport(format!("invalid upstream JSON: {error}"))
+ })?;
+ let agg = decode_aggregated_response(&body, wire_format)
+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;
+ LlmResponse::Agg(agg)
+ };
+
+ Ok(Response {
+ llm_response,
+ metadata,
+ })
+ }
+
+ /// The whole decode → call → encode path a wire endpoint needs, in one call.
+ ///
+ /// Decodes `raw_http_request` from `wire_format` to the neutral IR, serves it via
+ /// [`call_rewrite_model`](Self::call_rewrite_model) — the *upstream* wire format is
+ /// resolved there from the model's backend, independently of `wire_format` — then
+ /// encodes the neutral response back into `wire_format`. The result is a buffered
+ /// [`RawResponse::Buffered`] JSON body or a streamed [`RawResponse::Stream`] of
+ /// wire events (the caller frames the stream as SSE). The response's `model` is
+ /// restamped with the model the request asked for, never the upstream id.
+ ///
+ /// `http_headers` are carried through as the request's
+ /// [`Metadata::http_headers`] and forwarded to the upstream (minus the reserved
+ /// set); pass `None` to forward nothing.
+ pub async fn call_rewrite_model_raw(
+ &self,
+ ctx: Context,
+ raw_http_request: Value,
+ http_headers: Option<BTreeMap<String, String>>,
+ model: Option<&str>,
+ wire_format: WireFormat,
+ ) -> Result<RawResponse> {
+ let llm_request = decode_request(wire_format, &raw_http_request)
+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;
+ // The model the client asked for; restamped onto the response so it never
+ // leaks the upstream id.
+ let requested_model = llm_request.model.clone();
+
+ let request = Request {
+ llm_request,
+ raw_request: None,
+ metadata: Some(Metadata {
+ session_id: None,
+ agent_id: None,
+ task_id: None,
+ correlation_id: None,
+ extra_metadata: None,
+ http_headers,
+ wire_format: None,
+ }),
+ };
+ let response = self.call_rewrite_model(ctx, request, model).await?;
+
+ match response.llm_response {
+ LlmResponse::Agg(agg) => {
+ let body =
+ encode_aggregated_response(&agg, wire_format, requested_model.as_deref())
+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;
+ Ok(RawResponse::Buffered(body))
+ }
+ LlmResponse::Stream(chunks) => {
+ let events = encode_stream(chunks, wire_format, requested_model)
+ .map_err(|error| LlmClientError::Stream(error.to_string()))?;
+ Ok(RawResponse::Stream(events))
+ }
+ }
+ }
+}
+
+#[async_trait]
+impl RoutedLlmClient for TranslatingLlmClient {
+ async fn call(
+ &self,
+ ctx: Context,
+ request: Request,
+ decision: std::sync::Arc<dyn Decision>,
+ ) -> std::result::Result<Response, Box<dyn std::error::Error + Send + Sync>> {
+ let model_name = Some(decision.selected_model());
+ self.call_rewrite_model(ctx, request, model_name)
+ .await
+ .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>)
+ }
+}
+
+// Forwards caller-supplied metadata headers, skipping the reserved set.
+fn forward_metadata_headers(
+ mut builder: RequestBuilder,
+ metadata: Option<&Metadata>,
+) -> RequestBuilder {
+ let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
+ return builder;
+ };
+ for (name, value) in headers {
+ if is_reserved_header(name) {
+ continue;
+ }
+ builder = builder.header(name, value);
+ }
+ builder
+}
+
+// Adds the backend's static per-call headers.
+fn apply_extra_headers(mut builder: RequestBuilder, backend: &Backend) -> RequestBuilder {
+ for (name, value) in backend.extra_headers() {
+ builder = builder.header(name, value);
+ }
+ builder
+}
+
+// Overwrites the outbound body's `model` field with the resolved model id.
+fn set_json_model(body: &mut Value, model: &str) {
+ if let Value::Object(object) = body {
+ object.insert("model".to_string(), Value::String(model.to_string()));
+ }
+}
+
+// Case-insensitive membership test against RESERVED_HEADERS.
+fn is_reserved_header(name: &str) -> bool {
+ RESERVED_HEADERS
+ .iter()
+ .any(|reserved| name.eq_ignore_ascii_case(reserved))
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::BTreeMap;
+ use std::error::Error;
+
+ use serde_json::json;
+ use switchyard_protocol::{completion_text, text_request, LlmRequest};
+ use wiremock::matchers::{method, path};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+
+ use super::*;
+ use crate::backend::HttpBackendConfig;
+
+ fn config(base_url: &str) -> HttpBackendConfig {
+ HttpBackendConfig {
+ base_url: base_url.to_string(),
+ api_key: Some("secret".to_string()),
+ extra_headers: BTreeMap::new(),
+ }
+ }
+
+ // A one-model config list: "gpt" served over OpenAI Chat at base_url.
+ fn chat_map(base_url: &str) -> Vec<ModelConfig> {
+ vec![ModelConfig::new(
+ "gpt",
+ Backend::OpenAiChat(config(base_url)),
+ None,
+ )]
+ }
+
+ fn request_for(model: Option<&str>, stream: bool) -> Request {
+ let mut llm_request = text_request(model.map(str::to_string), "hi");
+ llm_request.stream = stream;
+ Request {
+ llm_request,
+ raw_request: None,
+ metadata: None,
+ }
+ }
+
+ // A request that pins `format` in its metadata, so the client resolves that
+ // wire format instead of the model's default backend.
+ fn request_with_wire_format(model: &str, format: WireFormat) -> Request {
+ let mut request = request_for(Some(model), false);
+ request.metadata = Some(Metadata {
+ session_id: None,
+ agent_id: None,
+ task_id: None,
+ correlation_id: None,
+ extra_metadata: None,
+ http_headers: None,
+ wire_format: Some(format),
+ });
+ request
+ }
+
+ #[tokio::test]
+ async fn missing_model_errors(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let client = TranslatingLlmClient::new(&[])?;
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(None, false), None)
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(error, LlmClientError::MissingModel));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn unknown_model_errors(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let client = TranslatingLlmClient::new(&[])?;
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "gpt"));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn unknown_model_format_errors(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ // "gpt" exists but only over OpenAI Chat; the request pins Anthropic.
+ let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
+ let Err(error) = client
+ .call_rewrite_model(
+ Context::default(),
+ request_with_wire_format("gpt", WireFormat::AnthropicMessages),
+ None,
+ )
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(
+ error,
+ LlmClientError::UnknownModelFormat { model, format }
+ if model == "gpt" && format == WireFormat::AnthropicMessages
+ ));
+ Ok(())
+ }
+
+ #[test]
+ fn backend_for_resolves_configured_format(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;
+ // "gpt" is served over OpenAI Chat only; other formats and models miss.
+ assert!(client.backend_for("gpt", WireFormat::OpenAiChat).is_some());
+ assert!(client
+ .backend_for("gpt", WireFormat::AnthropicMessages)
+ .is_none());
+ assert!(client
+ .backend_for("missing", WireFormat::OpenAiChat)
+ .is_none());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn model_name_arg_wins_over_request_model(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let client = TranslatingLlmClient::new(&[])?;
+ // Arg "b" is looked up (and reported), not the request's "a".
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("a"), false), Some("b"))
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "b"));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn buffered_openai_chat_round_trips(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "chatcmpl-1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "Hi there"},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await?;
+ let agg = response.llm_response.into_agg().await?;
+ assert_eq!(completion_text(&agg), "Hi there");
+
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn rewrites_model_to_resolved_upstream_id(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ // Inbound body says "switchyard"; the upstream must receive "gpt".
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1", "model": "gpt",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ // Inbound model differs from the map key / resolved model.
+ client
+ .call_rewrite_model(
+ Context::default(),
+ request_for(Some("switchyard"), false),
+ Some("gpt"),
+ )
+ .await?;
+ // The body_partial_json matcher asserts the upstream saw model "gpt".
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn streaming_openai_chat_aggregates(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
+ data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
+ data: [DONE]\n\n";
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)
+ .await?;
+ assert!(matches!(response.llm_response, LlmResponse::Stream(_)));
+ let agg = response.llm_response.into_agg().await?;
+ assert_eq!(completion_text(&agg), "Hello world");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn upstream_500_is_upstream_http(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(500).set_body_string("boom"))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(
+ error,
+ LlmClientError::UpstreamHttp { status: 500, .. }
+ ));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn context_overflow_400_is_mapped(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .respond_with(ResponseTemplate::new(400).set_body_json(json!({
+ "error": {"code": "context_length_exceeded", "message": "too big"}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected an error");
+ };
+ assert!(matches!(
+ error,
+ LlmClientError::ContextWindowExceeded { model, .. } if model == "gpt"
+ ));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn forwards_metadata_headers_except_reserved(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(wiremock::matchers::header("x-request-id", "abc"))
+ // A forwarded Authorization must NOT override the backend's bearer key.
+ .and(wiremock::matchers::header("authorization", "Bearer secret"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1", "model": "gpt",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+
+ let mut headers = BTreeMap::new();
+ headers.insert("x-request-id".to_string(), "abc".to_string());
+ headers.insert("authorization".to_string(), "Bearer client-key".to_string());
+ let request = Request {
+ llm_request: LlmRequest {
+ model: Some("gpt".to_string()),
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: Some(Metadata {
+ session_id: None,
+ agent_id: None,
+ task_id: None,
+ correlation_id: None,
+ extra_metadata: None,
+ http_headers: Some(headers),
+ wire_format: None,
+ }),
+ };
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+
+ // Matchers assert forwarded x-request-id survives and reserved
+ // authorization is the backend's, not the client's.
+ client
+ .call_rewrite_model(Context::default(), request, None)
+ .await?;
+ Ok(())
+ }
+
+ // Minimal `Decision` for driving the client through the `RoutedLlmClient` trait.
+ struct FixedDecision(&'static str);
+
+ impl Decision for FixedDecision {
+ fn selected_model(&self) -> &str {
+ self.0
+ }
+ fn reasoning(&self) -> Option<&str> {
+ None
+ }
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+ }
+
+ // Exercises the `RoutedLlmClient` impl: `call` resolves the upstream model from the
+ // decision (the request carries none) and round-trips a buffered response.
+ #[tokio::test]
+ async fn routed_llm_client_serves_the_decision_model(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "chatcmpl-1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "routed hi"},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let decision: std::sync::Arc<dyn Decision> = std::sync::Arc::new(FixedDecision("gpt"));
+ // Called through the trait; the request has no model, so "gpt" comes from the decision.
+ let response = client
+ .call(Context::default(), request_for(None, false), decision)
+ .await?;
+ let agg = response.llm_response.into_agg().await?;
+ assert_eq!(completion_text(&agg), "routed hi");
+ Ok(())
+ }
+
+ // Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI
+ // Chat JSON, with the client-facing `model` restamped over the upstream id.
+ #[tokio::test]
+ async fn call_rewrite_model_raw_round_trips_buffered_json(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "chatcmpl-1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "Hi there"},
+ "finish_reason": "stop"
+ }],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}
+ })))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let raw = json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}]
+ });
+ let RawResponse::Buffered(body) = client
+ .call_rewrite_model_raw(
+ Context::default(),
+ raw,
+ None,
+ Some("gpt"),
+ WireFormat::OpenAiChat,
+ )
+ .await?
+ else {
+ panic!("expected a buffered response");
+ };
+
+ assert_eq!(body["choices"][0]["message"]["content"], "Hi there");
+ // The client sees the model it asked for, not the upstream "gpt".
+ assert_eq!(body["model"], "client-facing");
+ Ok(())
+ }
+
+ // Raw path, streaming: an inbound `stream: true` request yields an unframed stream
+ // of OpenAI Chat chunk objects whose deltas reassemble the completion.
+ #[tokio::test]
+ async fn call_rewrite_model_raw_streams_wire_events(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ use futures::TryStreamExt;
+
+ let server = MockServer::start().await;
+ let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
+ data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
+ data: [DONE]\n\n";
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
+ .mount(&server)
+ .await;
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let raw = json!({
+ "model": "client-facing",
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": true
+ });
+ let RawResponse::Stream(stream) = client
+ .call_rewrite_model_raw(
+ Context::default(),
+ raw,
+ None,
+ Some("gpt"),
+ WireFormat::OpenAiChat,
+ )
+ .await?
+ else {
+ panic!("expected a streamed response");
+ };
+
+ let events: Vec<Value> = stream.try_collect().await?;
+ assert!(!events.is_empty(), "expected at least one wire event");
+ let content: String = events
+ .iter()
+ .filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
+ .collect();
+ assert_eq!(content, "Hello world");
+ Ok(())
+ }
+
+ // Raw path forwards caller headers (minus the reserved set) to the upstream.
+ #[tokio::test]
+ async fn call_rewrite_model_raw_forwards_headers(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(wiremock::matchers::header("x-request-id", "abc"))
+ // A forwarded authorization must NOT override the backend's bearer key.
+ .and(wiremock::matchers::header("authorization", "Bearer secret"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "1", "model": "gpt",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+ "usage": {}
+ })))
+ .mount(&server)
+ .await;
+
+ let mut headers = BTreeMap::new();
+ headers.insert("x-request-id".to_string(), "abc".to_string());
+ headers.insert("authorization".to_string(), "Bearer client-key".to_string());
+
+ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});
+ // Matchers assert the forwarded x-request-id survives and reserved
+ // authorization is the backend's, not the client's.
+ client
+ .call_rewrite_model_raw(
+ Context::default(),
+ raw,
+ Some(headers),
+ Some("gpt"),
+ WireFormat::OpenAiChat,
+ )
+ .await?;
+ Ok(())
+ }
+}
diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rs
new file mode 100644
index 000000000..d02c0523c
--- /dev/null
+++ b/crates/libsy-llm-client/src/error.rs
@@ -0,0 +1,157 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Error type for the LLM client, plus shared context-window-overflow detection.
+//!
+//! The overflow detection is ported from
+//! `switchyard-components/src/backends/context_overflow.rs`; that helper is
+//! crate-private there and this crate cannot depend on `switchyard-components`,
+//! so the small, self-contained logic is vendored here.
+
+use serde_json::Value;
+use switchyard_translation::WireFormat;
+use thiserror::Error;
+
+/// Result alias for LLM client operations.
+pub type Result<T> = std::result::Result<T, LlmClientError>;
+
+/// Failures surfaced while resolving, translating, sending, or decoding a call.
+#[derive(Debug, Error)]
+pub enum LlmClientError {
+ /// The resolved model name has no entry in the client's backend map.
+ #[error("no backend configured for model {0:?}")]
+ UnknownModel(String),
+
+ /// The model exists but has no backend configured for the requested format.
+ #[error("model {model:?} has no backend for format {format}")]
+ UnknownModelFormat {
+ /// Model that was resolved.
+ model: String,
+ /// Wire format requested for the call.
+ format: WireFormat,
+ },
+
+ /// Neither an explicit `model_name` nor a model on the request was given.
+ #[error("no model given: pass model_name or set request.llm_request.model")]
+ MissingModel,
+
+ /// Request encoding or response decoding failed in the translation engine.
+ #[error("translation failed: {0}")]
+ Translation(String),
+
+ /// The HTTP request could not be sent (connect/timeout/transport failure).
+ #[error("upstream transport error: {0}")]
+ Transport(String),
+
+ /// An upstream 400 whose body is detected as a context-window overflow.
+ ///
+ /// Kept distinct from [`LlmClientError::UpstreamHttp`] and checked first so a
+ /// caller can implement evict-and-retry by matching this variant instead of
+ /// sniffing error strings.
+ #[error("context window exceeded for model {model}: {message}")]
+ ContextWindowExceeded {
+ /// Model that overflowed, as sent upstream.
+ model: String,
+ /// Raw upstream error body.
+ message: String,
+ },
+
+ /// A non-2xx upstream response that is not a context-window overflow.
+ #[error("upstream returned HTTP {status}: {body}")]
+ UpstreamHttp {
+ /// Upstream HTTP status code.
+ status: u16,
+ /// Raw upstream error body.
+ body: String,
+ },
+
+ /// A streamed response failed mid-flight (read error or malformed frame).
+ #[error("stream error: {0}")]
+ Stream(String),
+}
+
+/// Detects a context-overflow body using a provider-supplied structured check
+/// and a substring phrase list.
+///
+/// Parses the body once, runs the structured check (e.g. against `error.code`),
+/// then falls back to matching phrases against `error.message` or — when the
+/// body is not JSON — the raw body. Centralizing the shape means each new
+/// provider-wrap of the canonical error is a one-line phrase entry, not a fork
+/// of the parsing logic.
+pub(crate) fn is_overflow_body<F>(body: &str, structured_check: F, phrases: &[&str]) -> bool
+where
+ F: Fn(&Value) -> bool,
+{
+ if let Ok(value) = serde_json::from_str::<Value>(body) {
+ if structured_check(&value) {
+ return true;
+ }
+ if let Some(message) = value
+ .get("error")
+ .and_then(|err| err.get("message"))
+ .and_then(Value::as_str)
+ {
+ if contains_any(message, phrases) {
+ return true;
+ }
+ }
+ }
+ // Some upstream proxies return plain-text bodies; fall through to a string
+ // match on the raw body.
+ contains_any(body, phrases)
+}
+
+// Case-insensitive substring match of any phrase against the message.
+fn contains_any(message: &str, phrases: &[&str]) -> bool {
+ let lower = message.to_ascii_lowercase();
+ phrases.iter().any(|phrase| lower.contains(phrase))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const PHRASES: &[&str] = &["context window", "too long"];
+
+ fn never(_value: &Value) -> bool {
+ false
+ }
+
+ #[test]
+ fn structured_check_short_circuits() {
+ let body = r#"{"error":{"code":"context_length_exceeded","message":"unrelated"}}"#;
+ let matched = is_overflow_body(
+ body,
+ |value| {
+ value
+ .get("error")
+ .and_then(|err| err.get("code"))
+ .and_then(Value::as_str)
+ == Some("context_length_exceeded")
+ },
+ &[],
+ );
+ assert!(matched);
+ }
+
+ #[test]
+ fn falls_back_to_message_phrase_match() {
+ let body = r#"{"error":{"message":"prompt too long"}}"#;
+ assert!(is_overflow_body(body, never, PHRASES));
+ }
+
+ #[test]
+ fn matches_plain_text_body() {
+ assert!(is_overflow_body(
+ "plain text mentioning context window",
+ never,
+ PHRASES
+ ));
+ }
+
+ #[test]
+ fn non_match_returns_false() {
+ let body = r#"{"error":{"message":"rate limit exceeded"}}"#;
+ assert!(!is_overflow_body(body, never, PHRASES));
+ }
+}
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
new file mode 100644
index 000000000..22a1c4726
--- /dev/null
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! HTTP LLM client that speaks Switchyard's neutral IR directly.
+//!
+//! [`TranslatingLlmClient`] maps a model name (and the wire format resolved from
+//! the request) to a [`Backend`],
+//! encodes a [`switchyard_protocol::Request`] to that backend's wire format via
+//! `switchyard-translation`, applies auth and forwards caller headers, makes the
+//! HTTP call with a shared [`reqwest::Client`], and decodes the wire response
+//! back to a [`switchyard_protocol::Response`] — supporting both buffered and
+//! streamed responses.
+
+pub mod backend;
+pub mod client;
+pub mod error;
+pub mod raw;
+
+pub use backend::{Backend, HttpBackendConfig};
+pub use client::{ModelConfig, TranslatingLlmClient};
+pub use error::{LlmClientError, Result};
+pub use raw::RawResponse;
+pub use switchyard_translation::RawEventStream;
diff --git a/crates/libsy-llm-client/src/raw.rs b/crates/libsy-llm-client/src/raw.rs
new file mode 100644
index 000000000..7391c1a0d
--- /dev/null
+++ b/crates/libsy-llm-client/src/raw.rs
@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! The raw wire result of a call.
+//!
+//! The encoders that produce it — buffered JSON and the streamed wire-event
+//! encoder — live in `switchyard-translation` alongside the rest of the neutral-IR
+//! codecs; this module only holds the result type they feed.
+
+use serde_json::Value;
+use switchyard_translation::RawEventStream;
+
+/// The wire result of
+/// [`call_rewrite_model_raw`](crate::TranslatingLlmClient::call_rewrite_model_raw):
+/// a buffered JSON body, or a live stream of wire events — both already in the
+/// requested wire format.
+pub enum RawResponse {
+ /// A complete JSON response body.
+ Buffered(Value),
+ /// A live stream of wire-format event objects, ready for SSE framing.
+ Stream(RawEventStream),
+}