Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(switchyard-llm-client): add HTTP LLM client crate (#79)
Greg Clark0bc032adb27f
diff --git a/Cargo.lock b/Cargo.lockindex b432a6d95..9662c3551 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2,6 +2,15 @@# It is not intended for manual editing.version = 4+[[package]]+name = "aho-corasick"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"+dependencies = [+ "memchr",+]+[[package]]name = "allocator-api2"version = "0.2.21"@@ -67,6 +76,16 @@ dependencies = ["rustversion",]+[[package]]+name = "assert-json-diff"+version = "2.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"+dependencies = [+ "serde",+ "serde_json",+]+[[package]]name = "async-stream"version = "0.3.6"@@ -348,6 +367,24 @@ dependencies = ["libc",]+[[package]]+name = "deadpool"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b"+dependencies = [+ "deadpool-runtime",+ "lazy_static",+ "num_cpus",+ "tokio",+]++[[package]]+name = "deadpool-runtime"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"+[[package]]name = "diff"version = "0.1.13"@@ -593,6 +630,12 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"+[[package]]+name = "hermit-abi"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"+[[package]]name = "http"version = "1.4.2"@@ -851,6 +894,12 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "lazy_static"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"+[[package]]name = "libc"version = "0.2.186"@@ -940,6 +989,16 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "num_cpus"+version = "1.17.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"+dependencies = [+ "hermit-abi",+ "libc",+]+[[package]]name = "once_cell"version = "1.21.4"@@ -1260,6 +1319,35 @@ dependencies = ["bitflags",]+[[package]]+name = "regex"+version = "1.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.4.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"+[[package]]name = "reqwest"version = "0.12.28"@@ -1629,6 +1717,22 @@ dependencies = ["tokio-stream",]+[[package]]+name = "switchyard-llm-client"+version = "0.1.0"+dependencies = [+ "async-trait",+ "futures",+ "futures-util",+ "reqwest",+ "serde_json",+ "switchyard-protocol",+ "switchyard-translation",+ "thiserror",+ "tokio",+ "wiremock",+]+[[package]]name = "switchyard-protocol"version = "0.1.0"@@ -2199,6 +2303,29 @@ dependencies = ["memchr",]+[[package]]+name = "wiremock"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031"+dependencies = [+ "assert-json-diff",+ "base64",+ "deadpool",+ "futures",+ "http",+ "http-body-util",+ "hyper",+ "hyper-util",+ "log",+ "once_cell",+ "regex",+ "serde",+ "serde_json",+ "tokio",+ "url",+]+[[package]]name = "writeable"version = "0.6.3"diff --git a/Cargo.toml b/Cargo.tomlindex ec2ee2c7a..551d52cfa 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -6,6 +6,7 @@ resolver = "2"members = ["crates/libsy","crates/libsy-examples",+ "crates/libsy-llm-client","crates/switchyard-components","crates/switchyard-components-v2","crates/switchyard-components-v2-macros",diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlnew file mode 100644index 000000000..67c338931--- /dev/null+++ b/crates/libsy-llm-client/Cargo.toml@@ -0,0 +1,26 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "switchyard-llm-client"+version = "0.1.0"+description = "HTTP LLM client speaking Switchyard's neutral IR directly"+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true++[dependencies]+switchyard-protocol = { path = "../protocol", version = "0.1.0" }+switchyard-translation = { path = "../switchyard-translation", version = "0.1.0" }+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }+async-trait = "0.1"+futures-util = "0.3"+serde_json = "1"+thiserror = "2"++[dev-dependencies]+futures = "0.3"+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }+wiremock = "0.6"diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdnew file mode 100644index 000000000..e769123f6--- /dev/null+++ b/crates/libsy-llm-client/README.md@@ -0,0 +1,193 @@+<!--+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+SPDX-License-Identifier: Apache-2.0+-->++# libsy-llm-client++An HTTP client that speaks Switchyard's neutral IR directly. You hand it a+[`switchyard_protocol::Request`] and a model name; it looks up the configured backend,+encodes the request to that backend's wire format, adds auth and forwards your+headers, makes the call with a shared `reqwest::Client`, and decodes the reply+back into a [`switchyard_protocol::Response`] — buffered or streamed.++It depends only on `switchyard-protocol` and `switchyard-translation`; no server, no+provider SDK.++## Concepts++- **Model configs.** A client is built from [`ModelConfig`] values. Each model has a+ default [`Backend`] and can have additional backends for other wire formats.+ [`TranslatingLlmClient::call_rewrite_model`] uses the request's metadata wire format+ when set, otherwise the model's default backend.+- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or+ `Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,+ static `extra_headers`). The variant fixes the URL path and auth scheme+ (Bearer vs `x-api-key` + `anthropic-version`).+- **Model rewrite.** The resolved model name is both the map key and the model id+ sent upstream — it overwrites whatever `model` the request arrived with.+- **Streaming is chosen by the request.** If the encoded body has `stream: true`+ (i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise+ `LlmResponse::Agg`.++## Add the dependency++Within this workspace:++```toml+[dependencies]+switchyard-llm-client = { path = "../libsy-llm-client" }+switchyard-protocol = { path = "../libsy-protocol" }+switchyard-translation = { path = "../switchyard-translation" } # for WireFormat+```++## Quickstart++### Build a client++```rust+use std::collections::BTreeMap;+use switchyard_llm_client::{+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient,+};++fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {+ let openai = HttpBackendConfig {+ base_url: "https://api.openai.com/v1".to_string(),+ api_key: std::env::var("OPENAI_API_KEY").ok(),+ extra_headers: BTreeMap::new(),+ };++ let models = [ModelConfig::new(+ "gpt-4o-mini",+ Backend::OpenAiChat(openai),+ None,+ )];++ TranslatingLlmClient::new(&models)+}+```++### Buffered call++```rust+use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};+use switchyard_protocol::{completion_text, text_request, Context, LlmResponse, Request};++async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<String> {+ let request = Request {+ llm_request: text_request(None, "Say hello in five words."),+ raw_request: None,+ metadata: None,+ };++ // model_name wins over request.llm_request.model; it is also sent upstream.+ let response = client+ .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))+ .await?;++ match response.llm_response {+ LlmResponse::Agg(agg) => Ok(completion_text(&agg)),+ LlmResponse::Stream(_) => Err(LlmClientError::Stream(+ "expected a buffered response".to_string(),+ )),+ }+}+```++### Streaming call++Set `stream` on the IR request and drive the returned chunk stream:++```rust+use futures_util::StreamExt;+use switchyard_llm_client::{LlmClientError, TranslatingLlmClient};+use switchyard_protocol::{text_request, Context, LlmResponse, LlmResponseChunk, Request};++async fn stream(client: &TranslatingLlmClient) -> switchyard_llm_client::Result<()> {+ let mut llm_request = text_request(None, "Count to five.");+ llm_request.stream = true;+ let request = Request { llm_request, raw_request: None, metadata: None };++ let response = client+ .call_rewrite_model(Context::default(), request, Some("gpt-4o-mini"))+ .await?;++ if let LlmResponse::Stream(mut chunks) = response.llm_response {+ while let Some(item) = chunks.next().await {+ match item {+ Ok(LlmResponseChunk::TextDelta { text, .. }) => print!("{text}"),+ Ok(_) => {} // usage, tool-call deltas, message start/stop+ Err(error) => return Err(LlmClientError::Stream(error.to_string())),+ }+ }+ }+ Ok(())+}+```++## Cross-format translation++The request/response are translated through the neutral IR, so the inbound shape+you build and the backend's wire format are independent. Pointing a+`WireFormat::AnthropicMessages` backend at an Anthropic endpoint while building+requests with the same helpers works the same way. Set `request.metadata.wire_format`+to select a non-default backend before calling `call_rewrite_model`. Register several+formats under one model to serve it over more than one upstream API:++```rust+use switchyard_llm_client::{+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient,+};++fn build_multi_format_client(+ openai_chat: HttpBackendConfig,+ openai_responses: HttpBackendConfig,+ anthropic: HttpBackendConfig,+) -> switchyard_llm_client::Result<TranslatingLlmClient> {+ let models = [ModelConfig::new(+ "my-model",+ Backend::OpenAiChat(openai_chat),+ Some(vec![+ Backend::OpenAiResponses(openai_responses),+ Backend::Anthropic(anthropic),+ ]),+ )];++ TranslatingLlmClient::new(&models)+}+```++## Headers & auth++- The `Backend` variant sets auth: OpenAI formats send `Authorization: Bearer <key>`;+ Anthropic sends `x-api-key: <key>` plus `anthropic-version`.+- `request.metadata.http_headers` are forwarded upstream, **except** reserved ones:+ `host`, `content-length`, `connection`, and the backend-owned+ `authorization` / `x-api-key` / `anthropic-version` / `content-type`. So a+ caller's placeholder credential never overrides the backend's real key.+- Per-backend static headers go in `HttpBackendConfig::extra_headers`.++## Errors++`call_rewrite_model` returns [`LlmClientError`]:++| Variant | When |+|---------|------|+| `MissingModel` | no `model_name` arg and no `request.llm_request.model` |+| `UnknownModel(model)` | model has no backends configured |+| `UnknownModelFormat { model, format }` | model has no backend for that format |+| `Translation(msg)` | encode/decode failed in the translation engine |+| `Transport(msg)` | connect/timeout/transport failure |+| `ContextWindowExceeded { model, message }` | upstream 400 detected as a context overflow (checked before `UpstreamHttp`, so callers can evict-and-retry) |+| `UpstreamHttp { status, body }` | any other non-2xx upstream response |+| `Stream(msg)` | mid-stream read / malformed frame |++[`switchyard_protocol::Request`]: ../libsy-protocol+[`switchyard_protocol::Response`]: ../libsy-protocol+[`libsy-proxy`]: ../libsy-proxy+[`Backend`]: src/backend.rs+[`HttpBackendConfig`]: src/backend.rs+[`ModelConfig`]: src/client.rs+[`TranslatingLlmClient::call_rewrite_model`]: src/client.rs+[`LlmClientError`]: src/error.rsdiff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsnew file mode 100644index 000000000..c907f290d--- /dev/null+++ b/crates/libsy-llm-client/src/backend.rs@@ -0,0 +1,256 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Per-provider backend configuration: wire format, upstream URL, and auth.++use std::{collections::BTreeMap, fmt};++use reqwest::RequestBuilder;+use switchyard_protocol::WireFormat;++use crate::error::is_overflow_body;++const ANTHROPIC_VERSION: &str = "2023-06-01";++// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new+// provider-wrap is a one-line entry here, not a fork of the parsing logic.+const OPENAI_OVERFLOW_PHRASES: &[&str] = &[+ "maximum context length",+ "context length exceeded",+ "context window",+ "context length is only",+ "please reduce the length of the input",+];++// Anthropic has no structured `error.code`, so detection is phrase-based only.+const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[+ "prompt is too long",+ "maximum number of tokens",+ "context window",+ "context length",+];++/// Shared HTTP configuration for one upstream backend.+#[derive(Clone)]+pub struct HttpBackendConfig {+ /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).+ pub base_url: String,+ /// API key for the provider, loaded by the caller. `None` sends no auth.+ pub api_key: Option<String>,+ /// Static headers added to every outbound call to this backend.+ pub extra_headers: BTreeMap<String, String>,+}++impl fmt::Debug for HttpBackendConfig {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("HttpBackendConfig")+ .field("base_url", &self.base_url)+ .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))+ .field("extra_headers", &self.extra_headers)+ .finish()+ }+}++/// A configured upstream backend, one variant per built-in wire format.+///+/// The variant fixes the wire format, URL path, and auth scheme together so no+/// invalid combination can be constructed.+#[derive(Clone, Debug)]+pub enum Backend {+ /// OpenAI-compatible Chat Completions API.+ OpenAiChat(HttpBackendConfig),+ /// OpenAI Responses API.+ OpenAiResponses(HttpBackendConfig),+ /// Anthropic Messages API.+ Anthropic(HttpBackendConfig),+}++impl Backend {+ /// The wire format the request IR is encoded to for this backend.+ pub fn wire_format(&self) -> WireFormat {+ match self {+ Backend::OpenAiChat(_) => WireFormat::OpenAiChat,+ Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,+ Backend::Anthropic(_) => WireFormat::AnthropicMessages,+ }+ }++ // Shared HTTP config, regardless of variant.+ fn config(&self) -> &HttpBackendConfig {+ match self {+ Backend::OpenAiChat(config)+ | Backend::OpenAiResponses(config)+ | Backend::Anthropic(config) => config,+ }+ }++ /// The fully resolved upstream URL for this backend's endpoint.+ ///+ /// Tolerates base URLs that already include the provider path (or a bare+ /// `/v1`), matching the join rules of the existing native backends.+ pub fn url(&self) -> String {+ let base_url = self.config().base_url.trim_end_matches('/');+ match self {+ Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),+ Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),+ Backend::Anthropic(_) => anthropic_url(base_url),+ }+ }++ /// Applies this backend's auth and version headers to a request builder.+ ///+ /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses+ /// `x-api-key: <key>` plus the required `anthropic-version` header.+ pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {+ let api_key = self.config().api_key.as_deref();+ match self {+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {+ if let Some(api_key) = api_key {+ builder = builder.bearer_auth(api_key);+ }+ }+ Backend::Anthropic(_) => {+ builder = builder.header("anthropic-version", ANTHROPIC_VERSION);+ if let Some(api_key) = api_key {+ builder = builder.header("x-api-key", api_key);+ }+ }+ }+ builder+ }++ /// Static per-backend headers to forward on every call.+ pub fn extra_headers(&self) -> &BTreeMap<String, String> {+ &self.config().extra_headers+ }++ /// Whether an upstream 400 `body` looks like a context-window overflow for+ /// this backend's provider.+ pub(crate) fn is_context_overflow(&self, body: &str) -> bool {+ match self {+ Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(+ body,+ |value| {+ value+ .get("error")+ .and_then(|err| err.get("code"))+ .and_then(serde_json::Value::as_str)+ == Some("context_length_exceeded")+ },+ OPENAI_OVERFLOW_PHRASES,+ ),+ Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),+ }+ }+}++// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.+fn openai_url(base_url: &str, suffix: &str) -> String {+ let base_root = base_url+ .strip_suffix("/chat/completions")+ .or_else(|| base_url.strip_suffix("/responses"))+ .unwrap_or(base_url);+ format!("{base_root}{suffix}")+}++// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.+fn anthropic_url(base_url: &str) -> String {+ if base_url.ends_with("/v1/messages") {+ base_url.to_string()+ } else if base_url.ends_with("/v1") {+ format!("{base_url}/messages")+ } else {+ format!("{base_url}/v1/messages")+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ fn config(base_url: &str) -> HttpBackendConfig {+ HttpBackendConfig {+ base_url: base_url.to_string(),+ api_key: Some("secret".to_string()),+ extra_headers: BTreeMap::new(),+ }+ }++ #[test]+ fn openai_chat_url_joins_bare_v1() {+ let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));+ assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");+ }++ #[test]+ fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {+ assert_eq!(+ Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),+ "https://api.openai.com/v1/chat/completions"+ );+ assert_eq!(+ Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),+ "https://api.openai.com/v1/chat/completions"+ );+ }++ #[test]+ fn openai_responses_url_uses_responses_path() {+ assert_eq!(+ Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),+ "https://api.openai.com/v1/responses"+ );+ }++ #[test]+ fn anthropic_url_join_cases() {+ assert_eq!(+ Backend::Anthropic(config("https://api.anthropic.com")).url(),+ "https://api.anthropic.com/v1/messages"+ );+ assert_eq!(+ Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),+ "https://api.anthropic.com/v1/messages"+ );+ assert_eq!(+ Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),+ "https://api.anthropic.com/v1/messages"+ );+ }++ #[test]+ fn wire_format_matches_variant() {+ assert_eq!(+ Backend::OpenAiChat(config("x")).wire_format(),+ WireFormat::OpenAiChat+ );+ assert_eq!(+ Backend::OpenAiResponses(config("x")).wire_format(),+ WireFormat::OpenAiResponses+ );+ assert_eq!(+ Backend::Anthropic(config("x")).wire_format(),+ WireFormat::AnthropicMessages+ );+ }++ #[test]+ fn openai_detects_canonical_and_wrapped_overflow() {+ let backend = Backend::OpenAiChat(config("x"));+ assert!(backend+ .is_context_overflow(r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#));+ // NVIDIA/LiteLLM message wrap with no structured code.+ assert!(backend.is_context_overflow(+ r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#+ ));+ assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));+ }++ #[test]+ fn anthropic_detects_prompt_too_long() {+ let backend = Backend::Anthropic(config("x"));+ assert!(backend+ .is_context_overflow(r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#));+ assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));+ }+}diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsnew file mode 100644index 000000000..a745ddef5--- /dev/null+++ b/crates/libsy-llm-client/src/client.rs@@ -0,0 +1,828 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral+//! request, call the configured backend over HTTP, decode the neutral response.++use std::collections::{BTreeMap, HashMap};++use async_trait::async_trait;+use futures_util::StreamExt;+use reqwest::RequestBuilder;+use serde_json::Value;+use switchyard_protocol::{+ Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient,+};+use switchyard_translation::{+ decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,+ encode_request, encode_stream, WireFormat,+};++use crate::backend::Backend;+use crate::error::{LlmClientError, Result};+use crate::raw::RawResponse;++// TODO: Why is this here? What does it do?+// Headers this client owns or that are hop-by-hop; never forwarded from the+// caller's metadata. Auth/version/content-type are set by the backend or the+// JSON body, so a forwarded copy would either be ignored or conflict. Compared+// case-insensitively. Aligns with `_SENSITIVE_HEADERS` in the Python+// `switchyard/lib/request_metadata.py` forwarding logic.+const RESERVED_HEADERS: &[&str] = &[+ "host",+ "content-length",+ "connection",+ "authorization",+ "proxy-authorization",+ "proxy-authenticate",+ "cookie",+ "set-cookie",+ "x-api-key",+ "anthropic-version",+ "content-type",+];++/// How one model is served: the `default_backend` used when the request does not+/// pin a wire format, plus any `other_backends` reachable over additional formats.+#[derive(Clone, Debug)]+pub struct ModelConfig {+ model_name: String,+ default_backend: Backend,+ other_backends: Option<Vec<Backend>>,+}++impl ModelConfig {+ /// A model named `model_name` served by `default_backend`, optionally reachable+ /// over additional wire formats via `other_backends`.+ pub fn new(+ model_name: impl Into<String>,+ default_backend: Backend,+ other_backends: Option<Vec<Backend>>,+ ) -> Self {+ Self {+ model_name: model_name.into(),+ default_backend,+ other_backends,+ }+ }+}++/// A client that dispatches neutral-IR requests to per-model HTTP backends.+///+/// Construct it with a list of [`ModelConfig`]s — one per model, each naming a+/// default [`Backend`] and any additional per-format backends. Each call resolves+/// the model and wire format, encodes the request to that backend's wire format,+/// applies auth and forwarded headers, sends the HTTP request with a shared+/// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered+/// or streamed).+pub struct TranslatingLlmClient {+ model_to_config: HashMap<String, ModelConfig>,+ client: reqwest::Client,+}++impl TranslatingLlmClient {+ /// Builds a client over the given [`ModelConfig`]s, with a fresh shared HTTP+ /// client and the built-in translation codecs.+ pub fn new(model_configs: &[ModelConfig]) -> Result<Self> {+ let client = reqwest::Client::builder().build().map_err(|error| {+ LlmClientError::Transport(format!("failed to build HTTP client: {error}"))+ })?;+ let model_to_config = model_configs+ .iter()+ .map(|config| (config.model_name.clone(), config.clone()))+ .collect();++ Ok(Self {+ model_to_config,+ client,+ })+ }++ /// The backend serving `model` over `format` — the default backend when its+ /// format matches, otherwise a matching entry in `other_backends`; `None` when+ /// the model is unknown or has no backend for `format`.+ pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> {+ self.model_to_config.get(model).and_then(|config| {+ if config.default_backend.wire_format() == format {+ Some(&config.default_backend)+ } else {+ config+ .other_backends+ .as_ref()+ .and_then(|backends| backends.iter().find(|b| b.wire_format() == format))+ }+ })+ }++ /// Calls the backend for `model_name` (or the request's own model), over the+ /// wire format the request pins in its metadata (else the model's default+ /// backend), and returns the neutral response.+ ///+ /// Resolution: `model_name` wins over `request.llm_request.model`; the+ /// resolved name is both the outer map key and the model id written into the+ /// request before translation. Errors with [`LlmClientError::MissingModel`]+ /// when neither is set, [`LlmClientError::UnknownModel`] when the model has+ /// no backends, and [`LlmClientError::UnknownModelFormat`] when the model has+ /// no backend for `format`.+ pub async fn call_rewrite_model(+ &self,+ _ctx: Context,+ request: Request,+ model_name: Option<&str>,+ ) -> Result<Response> {+ // Own the request's parts so the model can be set without a `mut` param+ // and without cloning the messages. `raw_request` is unused here.+ let Request {+ mut llm_request,+ metadata,+ ..+ } = request;++ let model = model_name+ .map(str::to_string)+ .or_else(|| llm_request.model.clone())+ .ok_or(LlmClientError::MissingModel)?;++ let orig_format = metadata.as_ref().and_then(|m| m.wire_format);+ let wire_format = orig_format.unwrap_or(+ self.model_to_config+ .get(&model)+ .map(|config| config.default_backend.wire_format())+ .ok_or(LlmClientError::UnknownModel(model.clone()))?,+ );+ let backend =+ self.backend_for(&model, wire_format)+ .ok_or(LlmClientError::UnknownModelFormat {+ model: model.clone(),+ format: wire_format,+ })?;++ // The resolved name is the upstream model id (per the crate contract).+ llm_request.model = Some(model.clone());++ let mut body = encode_request(&llm_request, wire_format)+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;+ // `encode_request` round-trips a preserved same-format body verbatim,+ // which keeps the caller's original `model`; force the resolved model so+ // the upstream always sees the target id.+ set_json_model(&mut body, &model);+ let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);++ let builder = self.client.post(backend.url()).json(&body);+ let builder = forward_metadata_headers(builder, metadata.as_ref());+ let builder = apply_extra_headers(builder, backend);+ let builder = backend.apply_auth(builder);++ let http_response = builder+ .send()+ .await+ .map_err(|error| LlmClientError::Transport(error.to_string()))?;+ let status = http_response.status();+ if !status.is_success() {+ let body = http_response+ .text()+ .await+ .unwrap_or_else(|error| format!("<failed to read error body: {error}>"));+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {+ return Err(LlmClientError::ContextWindowExceeded {+ model,+ message: body,+ });+ }+ return Err(LlmClientError::UpstreamHttp {+ status: status.as_u16(),+ body,+ });+ }++ let llm_response = if streaming {+ // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is+ // transport-agnostic and lives in `switchyard-translation`.+ let bytes = http_response.bytes_stream().map(|chunk| {+ chunk.map(|bytes| bytes.to_vec()).map_err(|error| {+ Box::new(LlmClientError::Stream(format!(+ "stream read failed: {error}"+ ))) as Box<dyn std::error::Error + Send + Sync>+ })+ });+ let chunks = decode_stream(bytes, wire_format)+ .map_err(|error| LlmClientError::Stream(error.to_string()))?;+ LlmResponse::Stream(chunks)+ } else {+ let body = http_response.json::<Value>().await.map_err(|error| {+ LlmClientError::Transport(format!("invalid upstream JSON: {error}"))+ })?;+ let agg = decode_aggregated_response(&body, wire_format)+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;+ LlmResponse::Agg(agg)+ };++ Ok(Response {+ llm_response,+ metadata,+ })+ }++ /// The whole decode → call → encode path a wire endpoint needs, in one call.+ ///+ /// Decodes `raw_http_request` from `wire_format` to the neutral IR, serves it via+ /// [`call_rewrite_model`](Self::call_rewrite_model) — the *upstream* wire format is+ /// resolved there from the model's backend, independently of `wire_format` — then+ /// encodes the neutral response back into `wire_format`. The result is a buffered+ /// [`RawResponse::Buffered`] JSON body or a streamed [`RawResponse::Stream`] of+ /// wire events (the caller frames the stream as SSE). The response's `model` is+ /// restamped with the model the request asked for, never the upstream id.+ ///+ /// `http_headers` are carried through as the request's+ /// [`Metadata::http_headers`] and forwarded to the upstream (minus the reserved+ /// set); pass `None` to forward nothing.+ pub async fn call_rewrite_model_raw(+ &self,+ ctx: Context,+ raw_http_request: Value,+ http_headers: Option<BTreeMap<String, String>>,+ model: Option<&str>,+ wire_format: WireFormat,+ ) -> Result<RawResponse> {+ let llm_request = decode_request(wire_format, &raw_http_request)+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;+ // The model the client asked for; restamped onto the response so it never+ // leaks the upstream id.+ let requested_model = llm_request.model.clone();++ let request = Request {+ llm_request,+ raw_request: None,+ metadata: Some(Metadata {+ session_id: None,+ agent_id: None,+ task_id: None,+ correlation_id: None,+ extra_metadata: None,+ http_headers,+ wire_format: None,+ }),+ };+ let response = self.call_rewrite_model(ctx, request, model).await?;++ match response.llm_response {+ LlmResponse::Agg(agg) => {+ let body =+ encode_aggregated_response(&agg, wire_format, requested_model.as_deref())+ .map_err(|error| LlmClientError::Translation(error.to_string()))?;+ Ok(RawResponse::Buffered(body))+ }+ LlmResponse::Stream(chunks) => {+ let events = encode_stream(chunks, wire_format, requested_model)+ .map_err(|error| LlmClientError::Stream(error.to_string()))?;+ Ok(RawResponse::Stream(events))+ }+ }+ }+}++#[async_trait]+impl RoutedLlmClient for TranslatingLlmClient {+ async fn call(+ &self,+ ctx: Context,+ request: Request,+ decision: std::sync::Arc<dyn Decision>,+ ) -> std::result::Result<Response, Box<dyn std::error::Error + Send + Sync>> {+ let model_name = Some(decision.selected_model());+ self.call_rewrite_model(ctx, request, model_name)+ .await+ .map_err(|error| Box::new(error) as Box<dyn std::error::Error + Send + Sync>)+ }+}++// Forwards caller-supplied metadata headers, skipping the reserved set.+fn forward_metadata_headers(+ mut builder: RequestBuilder,+ metadata: Option<&Metadata>,+) -> RequestBuilder {+ let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {+ return builder;+ };+ for (name, value) in headers {+ if is_reserved_header(name) {+ continue;+ }+ builder = builder.header(name, value);+ }+ builder+}++// Adds the backend's static per-call headers.+fn apply_extra_headers(mut builder: RequestBuilder, backend: &Backend) -> RequestBuilder {+ for (name, value) in backend.extra_headers() {+ builder = builder.header(name, value);+ }+ builder+}++// Overwrites the outbound body's `model` field with the resolved model id.+fn set_json_model(body: &mut Value, model: &str) {+ if let Value::Object(object) = body {+ object.insert("model".to_string(), Value::String(model.to_string()));+ }+}++// Case-insensitive membership test against RESERVED_HEADERS.+fn is_reserved_header(name: &str) -> bool {+ RESERVED_HEADERS+ .iter()+ .any(|reserved| name.eq_ignore_ascii_case(reserved))+}++#[cfg(test)]+mod tests {+ use std::collections::BTreeMap;+ use std::error::Error;++ use serde_json::json;+ use switchyard_protocol::{completion_text, text_request, LlmRequest};+ use wiremock::matchers::{method, path};+ use wiremock::{Mock, MockServer, ResponseTemplate};++ use super::*;+ use crate::backend::HttpBackendConfig;++ fn config(base_url: &str) -> HttpBackendConfig {+ HttpBackendConfig {+ base_url: base_url.to_string(),+ api_key: Some("secret".to_string()),+ extra_headers: BTreeMap::new(),+ }+ }++ // A one-model config list: "gpt" served over OpenAI Chat at base_url.+ fn chat_map(base_url: &str) -> Vec<ModelConfig> {+ vec![ModelConfig::new(+ "gpt",+ Backend::OpenAiChat(config(base_url)),+ None,+ )]+ }++ fn request_for(model: Option<&str>, stream: bool) -> Request {+ let mut llm_request = text_request(model.map(str::to_string), "hi");+ llm_request.stream = stream;+ Request {+ llm_request,+ raw_request: None,+ metadata: None,+ }+ }++ // A request that pins `format` in its metadata, so the client resolves that+ // wire format instead of the model's default backend.+ fn request_with_wire_format(model: &str, format: WireFormat) -> Request {+ let mut request = request_for(Some(model), false);+ request.metadata = Some(Metadata {+ session_id: None,+ agent_id: None,+ task_id: None,+ correlation_id: None,+ extra_metadata: None,+ http_headers: None,+ wire_format: Some(format),+ });+ request+ }++ #[tokio::test]+ async fn missing_model_errors(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let client = TranslatingLlmClient::new(&[])?;+ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(None, false), None)+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(error, LlmClientError::MissingModel));+ Ok(())+ }++ #[tokio::test]+ async fn unknown_model_errors(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let client = TranslatingLlmClient::new(&[])?;+ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "gpt"));+ Ok(())+ }++ #[tokio::test]+ async fn unknown_model_format_errors(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ // "gpt" exists but only over OpenAI Chat; the request pins Anthropic.+ let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;+ let Err(error) = client+ .call_rewrite_model(+ Context::default(),+ request_with_wire_format("gpt", WireFormat::AnthropicMessages),+ None,+ )+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(+ error,+ LlmClientError::UnknownModelFormat { model, format }+ if model == "gpt" && format == WireFormat::AnthropicMessages+ ));+ Ok(())+ }++ #[test]+ fn backend_for_resolves_configured_format(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?;+ // "gpt" is served over OpenAI Chat only; other formats and models miss.+ assert!(client.backend_for("gpt", WireFormat::OpenAiChat).is_some());+ assert!(client+ .backend_for("gpt", WireFormat::AnthropicMessages)+ .is_none());+ assert!(client+ .backend_for("missing", WireFormat::OpenAiChat)+ .is_none());+ Ok(())+ }++ #[tokio::test]+ async fn model_name_arg_wins_over_request_model(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let client = TranslatingLlmClient::new(&[])?;+ // Arg "b" is looked up (and reported), not the request's "a".+ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("a"), false), Some("b"))+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(error, LlmClientError::UnknownModel(model) if model == "b"));+ Ok(())+ }++ #[tokio::test]+ async fn buffered_openai_chat_round_trips(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "chatcmpl-1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "Hi there"},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;++ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await?;+ let agg = response.llm_response.into_agg().await?;+ assert_eq!(completion_text(&agg), "Hi there");++ Ok(())+ }++ #[tokio::test]+ async fn rewrites_model_to_resolved_upstream_id(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ // Inbound body says "switchyard"; the upstream must receive "gpt".+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(wiremock::matchers::body_partial_json(json!({"model": "gpt"})))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "1", "model": "gpt",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],+ "usage": {}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ // Inbound model differs from the map key / resolved model.+ client+ .call_rewrite_model(+ Context::default(),+ request_for(Some("switchyard"), false),+ Some("gpt"),+ )+ .await?;+ // The body_partial_json matcher asserts the upstream saw model "gpt".+ Ok(())+ }++ #[tokio::test]+ async fn streaming_openai_chat_aggregates(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\+ data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\+ data: [DONE]\n\n";+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;++ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)+ .await?;+ assert!(matches!(response.llm_response, LlmResponse::Stream(_)));+ let agg = response.llm_response.into_agg().await?;+ assert_eq!(completion_text(&agg), "Hello world");+ Ok(())+ }++ #[tokio::test]+ async fn upstream_500_is_upstream_http(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .respond_with(ResponseTemplate::new(500).set_body_string("boom"))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;++ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(+ error,+ LlmClientError::UpstreamHttp { status: 500, .. }+ ));+ Ok(())+ }++ #[tokio::test]+ async fn context_overflow_400_is_mapped(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .respond_with(ResponseTemplate::new(400).set_body_json(json!({+ "error": {"code": "context_length_exceeded", "message": "too big"}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;++ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected an error");+ };+ assert!(matches!(+ error,+ LlmClientError::ContextWindowExceeded { model, .. } if model == "gpt"+ ));+ Ok(())+ }++ #[tokio::test]+ async fn forwards_metadata_headers_except_reserved(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(wiremock::matchers::header("x-request-id", "abc"))+ // A forwarded Authorization must NOT override the backend's bearer key.+ .and(wiremock::matchers::header("authorization", "Bearer secret"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "1", "model": "gpt",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],+ "usage": {}+ })))+ .mount(&server)+ .await;++ let mut headers = BTreeMap::new();+ headers.insert("x-request-id".to_string(), "abc".to_string());+ headers.insert("authorization".to_string(), "Bearer client-key".to_string());+ let request = Request {+ llm_request: LlmRequest {+ model: Some("gpt".to_string()),+ ..LlmRequest::default()+ },+ raw_request: None,+ metadata: Some(Metadata {+ session_id: None,+ agent_id: None,+ task_id: None,+ correlation_id: None,+ extra_metadata: None,+ http_headers: Some(headers),+ wire_format: None,+ }),+ };++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;++ // Matchers assert forwarded x-request-id survives and reserved+ // authorization is the backend's, not the client's.+ client+ .call_rewrite_model(Context::default(), request, None)+ .await?;+ Ok(())+ }++ // Minimal `Decision` for driving the client through the `RoutedLlmClient` trait.+ struct FixedDecision(&'static str);++ impl Decision for FixedDecision {+ fn selected_model(&self) -> &str {+ self.0+ }+ fn reasoning(&self) -> Option<&str> {+ None+ }+ fn as_any(&self) -> &dyn std::any::Any {+ self+ }+ }++ // Exercises the `RoutedLlmClient` impl: `call` resolves the upstream model from the+ // decision (the request carries none) and round-trips a buffered response.+ #[tokio::test]+ async fn routed_llm_client_serves_the_decision_model(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "chatcmpl-1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "routed hi"},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let decision: std::sync::Arc<dyn Decision> = std::sync::Arc::new(FixedDecision("gpt"));+ // Called through the trait; the request has no model, so "gpt" comes from the decision.+ let response = client+ .call(Context::default(), request_for(None, false), decision)+ .await?;+ let agg = response.llm_response.into_agg().await?;+ assert_eq!(completion_text(&agg), "routed hi");+ Ok(())+ }++ // Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI+ // Chat JSON, with the client-facing `model` restamped over the upstream id.+ #[tokio::test]+ async fn call_rewrite_model_raw_round_trips_buffered_json(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "chatcmpl-1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "Hi there"},+ "finish_reason": "stop"+ }],+ "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3}+ })))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let raw = json!({+ "model": "client-facing",+ "messages": [{"role": "user", "content": "hi"}]+ });+ let RawResponse::Buffered(body) = client+ .call_rewrite_model_raw(+ Context::default(),+ raw,+ None,+ Some("gpt"),+ WireFormat::OpenAiChat,+ )+ .await?+ else {+ panic!("expected a buffered response");+ };++ assert_eq!(body["choices"][0]["message"]["content"], "Hi there");+ // The client sees the model it asked for, not the upstream "gpt".+ assert_eq!(body["model"], "client-facing");+ Ok(())+ }++ // Raw path, streaming: an inbound `stream: true` request yields an unframed stream+ // of OpenAI Chat chunk objects whose deltas reassemble the completion.+ #[tokio::test]+ async fn call_rewrite_model_raw_streams_wire_events(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ use futures::TryStreamExt;++ let server = MockServer::start().await;+ let sse = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\+ data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\+ data: [DONE]\n\n";+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))+ .mount(&server)+ .await;++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let raw = json!({+ "model": "client-facing",+ "messages": [{"role": "user", "content": "hi"}],+ "stream": true+ });+ let RawResponse::Stream(stream) = client+ .call_rewrite_model_raw(+ Context::default(),+ raw,+ None,+ Some("gpt"),+ WireFormat::OpenAiChat,+ )+ .await?+ else {+ panic!("expected a streamed response");+ };++ let events: Vec<Value> = stream.try_collect().await?;+ assert!(!events.is_empty(), "expected at least one wire event");+ let content: String = events+ .iter()+ .filter_map(|event| event["choices"][0]["delta"]["content"].as_str())+ .collect();+ assert_eq!(content, "Hello world");+ Ok(())+ }++ // Raw path forwards caller headers (minus the reserved set) to the upstream.+ #[tokio::test]+ async fn call_rewrite_model_raw_forwards_headers(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(wiremock::matchers::header("x-request-id", "abc"))+ // A forwarded authorization must NOT override the backend's bearer key.+ .and(wiremock::matchers::header("authorization", "Bearer secret"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "1", "model": "gpt",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],+ "usage": {}+ })))+ .mount(&server)+ .await;++ let mut headers = BTreeMap::new();+ headers.insert("x-request-id".to_string(), "abc".to_string());+ headers.insert("authorization".to_string(), "Bearer client-key".to_string());++ let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let raw = json!({"model": "gpt", "messages": [{"role": "user", "content": "hi"}]});+ // Matchers assert the forwarded x-request-id survives and reserved+ // authorization is the backend's, not the client's.+ client+ .call_rewrite_model_raw(+ Context::default(),+ raw,+ Some(headers),+ Some("gpt"),+ WireFormat::OpenAiChat,+ )+ .await?;+ Ok(())+ }+}diff --git a/crates/libsy-llm-client/src/error.rs b/crates/libsy-llm-client/src/error.rsnew file mode 100644index 000000000..d02c0523c--- /dev/null+++ b/crates/libsy-llm-client/src/error.rs@@ -0,0 +1,157 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Error type for the LLM client, plus shared context-window-overflow detection.+//!+//! The overflow detection is ported from+//! `switchyard-components/src/backends/context_overflow.rs`; that helper is+//! crate-private there and this crate cannot depend on `switchyard-components`,+//! so the small, self-contained logic is vendored here.++use serde_json::Value;+use switchyard_translation::WireFormat;+use thiserror::Error;++/// Result alias for LLM client operations.+pub type Result<T> = std::result::Result<T, LlmClientError>;++/// Failures surfaced while resolving, translating, sending, or decoding a call.+#[derive(Debug, Error)]+pub enum LlmClientError {+ /// The resolved model name has no entry in the client's backend map.+ #[error("no backend configured for model {0:?}")]+ UnknownModel(String),++ /// The model exists but has no backend configured for the requested format.+ #[error("model {model:?} has no backend for format {format}")]+ UnknownModelFormat {+ /// Model that was resolved.+ model: String,+ /// Wire format requested for the call.+ format: WireFormat,+ },++ /// Neither an explicit `model_name` nor a model on the request was given.+ #[error("no model given: pass model_name or set request.llm_request.model")]+ MissingModel,++ /// Request encoding or response decoding failed in the translation engine.+ #[error("translation failed: {0}")]+ Translation(String),++ /// The HTTP request could not be sent (connect/timeout/transport failure).+ #[error("upstream transport error: {0}")]+ Transport(String),++ /// An upstream 400 whose body is detected as a context-window overflow.+ ///+ /// Kept distinct from [`LlmClientError::UpstreamHttp`] and checked first so a+ /// caller can implement evict-and-retry by matching this variant instead of+ /// sniffing error strings.+ #[error("context window exceeded for model {model}: {message}")]+ ContextWindowExceeded {+ /// Model that overflowed, as sent upstream.+ model: String,+ /// Raw upstream error body.+ message: String,+ },++ /// A non-2xx upstream response that is not a context-window overflow.+ #[error("upstream returned HTTP {status}: {body}")]+ UpstreamHttp {+ /// Upstream HTTP status code.+ status: u16,+ /// Raw upstream error body.+ body: String,+ },++ /// A streamed response failed mid-flight (read error or malformed frame).+ #[error("stream error: {0}")]+ Stream(String),+}++/// Detects a context-overflow body using a provider-supplied structured check+/// and a substring phrase list.+///+/// Parses the body once, runs the structured check (e.g. against `error.code`),+/// then falls back to matching phrases against `error.message` or — when the+/// body is not JSON — the raw body. Centralizing the shape means each new+/// provider-wrap of the canonical error is a one-line phrase entry, not a fork+/// of the parsing logic.+pub(crate) fn is_overflow_body<F>(body: &str, structured_check: F, phrases: &[&str]) -> bool+where+ F: Fn(&Value) -> bool,+{+ if let Ok(value) = serde_json::from_str::<Value>(body) {+ if structured_check(&value) {+ return true;+ }+ if let Some(message) = value+ .get("error")+ .and_then(|err| err.get("message"))+ .and_then(Value::as_str)+ {+ if contains_any(message, phrases) {+ return true;+ }+ }+ }+ // Some upstream proxies return plain-text bodies; fall through to a string+ // match on the raw body.+ contains_any(body, phrases)+}++// Case-insensitive substring match of any phrase against the message.+fn contains_any(message: &str, phrases: &[&str]) -> bool {+ let lower = message.to_ascii_lowercase();+ phrases.iter().any(|phrase| lower.contains(phrase))+}++#[cfg(test)]+mod tests {+ use super::*;++ const PHRASES: &[&str] = &["context window", "too long"];++ fn never(_value: &Value) -> bool {+ false+ }++ #[test]+ fn structured_check_short_circuits() {+ let body = r#"{"error":{"code":"context_length_exceeded","message":"unrelated"}}"#;+ let matched = is_overflow_body(+ body,+ |value| {+ value+ .get("error")+ .and_then(|err| err.get("code"))+ .and_then(Value::as_str)+ == Some("context_length_exceeded")+ },+ &[],+ );+ assert!(matched);+ }++ #[test]+ fn falls_back_to_message_phrase_match() {+ let body = r#"{"error":{"message":"prompt too long"}}"#;+ assert!(is_overflow_body(body, never, PHRASES));+ }++ #[test]+ fn matches_plain_text_body() {+ assert!(is_overflow_body(+ "plain text mentioning context window",+ never,+ PHRASES+ ));+ }++ #[test]+ fn non_match_returns_false() {+ let body = r#"{"error":{"message":"rate limit exceeded"}}"#;+ assert!(!is_overflow_body(body, never, PHRASES));+ }+}diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsnew file mode 100644index 000000000..22a1c4726--- /dev/null+++ b/crates/libsy-llm-client/src/lib.rs@@ -0,0 +1,23 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! HTTP LLM client that speaks Switchyard's neutral IR directly.+//!+//! [`TranslatingLlmClient`] maps a model name (and the wire format resolved from+//! the request) to a [`Backend`],+//! encodes a [`switchyard_protocol::Request`] to that backend's wire format via+//! `switchyard-translation`, applies auth and forwards caller headers, makes the+//! HTTP call with a shared [`reqwest::Client`], and decodes the wire response+//! back to a [`switchyard_protocol::Response`] — supporting both buffered and+//! streamed responses.++pub mod backend;+pub mod client;+pub mod error;+pub mod raw;++pub use backend::{Backend, HttpBackendConfig};+pub use client::{ModelConfig, TranslatingLlmClient};+pub use error::{LlmClientError, Result};+pub use raw::RawResponse;+pub use switchyard_translation::RawEventStream;diff --git a/crates/libsy-llm-client/src/raw.rs b/crates/libsy-llm-client/src/raw.rsnew file mode 100644index 000000000..7391c1a0d--- /dev/null+++ b/crates/libsy-llm-client/src/raw.rs@@ -0,0 +1,22 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! The raw wire result of a call.+//!+//! The encoders that produce it — buffered JSON and the streamed wire-event+//! encoder — live in `switchyard-translation` alongside the rest of the neutral-IR+//! codecs; this module only holds the result type they feed.++use serde_json::Value;+use switchyard_translation::RawEventStream;++/// The wire result of+/// [`call_rewrite_model_raw`](crate::TranslatingLlmClient::call_rewrite_model_raw):+/// a buffered JSON body, or a live stream of wire events — both already in the+/// requested wire format.+pub enum RawResponse {+ /// A complete JSON response body.+ Buffered(Value),+ /// A live stream of wire-format event objects, ready for SSE framing.+ Stream(RawEventStream),+}