Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
26.1 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Per-provider backend configuration: wire format, upstream URL, and auth.

use std::time::Duration;
use std::{
    collections::{BTreeMap, BTreeSet},
    fmt,
};

use reqwest::RequestBuilder;
use reqwest::header::{HeaderName, HeaderValue};
use serde_json::Value;
use switchyard_protocol::{Metadata, WireFormat};

use crate::error::{LlmClientError, Result, is_overflow_body};

const ANTHROPIC_VERSION: &str = "2023-06-01";

/// Default number of retries for server-configured upstream calls.
pub const DEFAULT_MAX_RETRIES: u32 = 2;

// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
// provider-wrap is a one-line entry here, not a fork of the parsing logic.
const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
    "maximum context length",
    "context length exceeded",
    "context window",
    "context length is only",
    "please reduce the length of the input",
    "exceeds the maximum allowed input length",
    "exceeds the maximum allowed length",
    "exceeds the available context size",
    "is longer than the model's context length",
    // llama-server, both of its ERROR_TYPE_EXCEED_CONTEXT_SIZE messages
    // (tools/server/server-context.cpp). The structured check below catches
    // them from `error.type`; these cover a proxy that rewrites the envelope
    // and keeps the message.
    "exceeds the available context size",
    "larger than the max context size",
];

// Anthropic has no structured `error.code`, so detection is phrase-based only.
const ANTHROPIC_OVERFLOW_PHRASES: &[&str] = &[
    "prompt is too long",
    "maximum number of tokens",
    "context window",
    "context length",
];

/// Shared HTTP configuration for one upstream backend.
#[derive(Clone)]
pub struct HttpBackendConfig {
    /// Base URL of the provider API (e.g. `https://api.openai.com/v1`).
    pub base_url: String,
    /// API key for the provider, loaded by the caller. `None` sends no configured auth.
    /// Client construction rejects active values that cannot form the provider's auth header.
    pub api_key: Option<String>,
    /// Whether this backend forwards the caller's provider credential and application headers.
    ///
    /// The forwarding backends in one route must use one credential family (OpenAI or
    /// Anthropic) unless they all use the same scheme, host, and port, such as one LLM
    /// gateway. Backends that send a configured key are not restricted.
    pub forward_auth: bool,
    /// Custom headers added to every outbound call to this backend.
    ///
    /// Provider-owned headers are rejected so a static value cannot replace
    /// configured or forwarded auth. Names and values must be valid HTTP header bytes;
    /// header names are case-insensitive.
    /// Configured values replace caller values for the same header name.
    pub extra_headers: BTreeMap<String, String>,
    /// Default top-level request fields, applied only when the request omits the key.
    pub extra_body: BTreeMap<String, Value>,
    /// Top-level request fields dropped from the outbound body before `extra_body` is merged,
    /// so a target can reinstate one deliberately. For providers that reject an otherwise
    /// standard field (e.g. an internal API that 400s on `max_output_tokens`).
    pub omit_body_fields: BTreeSet<String>,
    /// Reasoning effort forced on every request to this backend, replacing whatever the caller
    /// sent. Responses carries it as `reasoning.effort`, Chat Completions as `reasoning_effort`,
    /// and Anthropic Messages as `output_config.effort`.
    pub reasoning_effort: Option<String>,
    /// Additional attempts for transient upstream failures.
    /// One template-compatibility retry may also run within the same timeout.
    pub max_retries: u32,
    /// Cooldown after an exhausted transient completion failure. Zero disables it.
    pub failure_cooldown: Duration,
    /// Deadline for one complete response, including retries, retry delays, and stream reads.
    /// `None` leaves the wait unbounded.
    pub timeout: Option<Duration>,
}

impl fmt::Debug for HttpBackendConfig {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("HttpBackendConfig")
            .field("base_url", &self.base_url)
            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
            .field("forward_auth", &self.forward_auth)
            .field("extra_header_names", &self.extra_headers.keys())
            .field("extra_body_keys", &self.extra_body.keys())
            .field("omit_body_fields", &self.omit_body_fields)
            .field("reasoning_effort", &self.reasoning_effort)
            .field("max_retries", &self.max_retries)
            .field("failure_cooldown", &self.failure_cooldown)
            .field("timeout", &self.timeout)
            .finish()
    }
}

/// A configured upstream backend, one variant per built-in wire format.
///
/// The variant fixes the wire format, URL path, and auth scheme together so no
/// invalid combination can be constructed.
#[derive(Clone, Debug)]
pub enum Backend {
    /// OpenAI-compatible Chat Completions API.
    OpenAiChat(HttpBackendConfig),
    /// OpenAI Responses API.
    OpenAiResponses(HttpBackendConfig),
    /// Anthropic Messages API.
    Anthropic(HttpBackendConfig),
}

impl Backend {
    // Matches reqwest's header conversions before the client can send a request.
    pub(crate) fn validate_configured_headers(&self, model_name: &str) -> Result<()> {
        for (name, value) in &self.config().extra_headers {
            if HeaderName::from_bytes(name.as_bytes()).is_err() {
                return Err(LlmClientError::Configuration {
                    message: format!(
                        "model {model_name:?} extra_headers contains invalid HTTP header name {name:?}"
                    ),
                });
            }
            if HeaderValue::from_bytes(value.as_bytes()).is_err() {
                return Err(LlmClientError::Configuration {
                    message: format!(
                        "model {model_name:?} has invalid HTTP header value for extra_headers entry {name:?}"
                    ),
                });
            }
        }

        let invalid_name = self.config().extra_headers.keys().find(|name| match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
                name.eq_ignore_ascii_case("authorization")
                    || (self.is_forwarding_auth()
                        && (name.eq_ignore_ascii_case("chatgpt-account-id")
                            || name.eq_ignore_ascii_case("x-openai-fedramp")))
            }
            Backend::Anthropic(_) => {
                name.eq_ignore_ascii_case("x-api-key")
                    || name.eq_ignore_ascii_case("anthropic-version")
                    || (self.is_forwarding_auth()
                        && (name.eq_ignore_ascii_case("authorization")
                            || name.eq_ignore_ascii_case("anthropic-beta")))
            }
        });
        if let Some(name) = invalid_name {
            return Err(LlmClientError::Configuration {
                message: format!(
                    "model {model_name:?} extra_headers cannot set {name:?}; extra_headers is only for additional headers"
                ),
            });
        }

        let Some(api_key) = self.configured_api_key() else {
            return Ok(());
        };
        let valid_api_key = match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
                HeaderValue::try_from(format!("Bearer {api_key}")).is_ok()
            }
            Backend::Anthropic(_) => HeaderValue::from_str(api_key).is_ok(),
        };
        if !valid_api_key {
            return Err(LlmClientError::Configuration {
                message: format!(
                    "model {model_name:?} api_key cannot be encoded as an HTTP header"
                ),
            });
        }
        Ok(())
    }

    /// The wire format the request IR is encoded to for this backend.
    pub fn wire_format(&self) -> WireFormat {
        match self {
            Backend::OpenAiChat(_) => WireFormat::OpenAiChat,
            Backend::OpenAiResponses(_) => WireFormat::OpenAiResponses,
            Backend::Anthropic(_) => WireFormat::AnthropicMessages,
        }
    }

    // Shared HTTP config, regardless of variant.
    fn config(&self) -> &HttpBackendConfig {
        match self {
            Backend::OpenAiChat(config)
            | Backend::OpenAiResponses(config)
            | Backend::Anthropic(config) => config,
        }
    }

    // Static credentials are unused when the caller's authorization is forwarded.
    fn configured_api_key(&self) -> Option<&str> {
        if self.is_forwarding_auth() {
            None
        } else {
            self.config().api_key.as_deref()
        }
    }

    pub(crate) fn base_url(&self) -> &str {
        &self.config().base_url
    }

    /// The fully resolved upstream URL for this backend's endpoint.
    ///
    /// Tolerates base URLs that already include the provider path (or a bare
    /// `/v1`), matching the join rules of the existing native backends.
    pub fn url(&self) -> String {
        let base_url = &self.config().base_url;
        let result = match self {
            Backend::OpenAiChat(_) => openai_url(base_url, "/chat/completions"),
            Backend::OpenAiResponses(_) => openai_url(base_url, "/responses"),
            Backend::Anthropic(_) => anthropic_url(base_url, ""),
        };
        result.unwrap_or_else(|error| {
            tracing::error!(%error, "Unable to build provider endpoint URL");
            base_url.clone()
        })
    }

    /// Applies this backend's configured auth and version headers to a request builder.
    ///
    /// OpenAI variants use `Authorization: Bearer <key>`; Anthropic uses
    /// `x-api-key: <key>` plus the required `anthropic-version` header. A backend
    /// with `forward_auth` uses the caller's provider credential instead.
    pub fn apply_auth(&self, mut builder: RequestBuilder) -> RequestBuilder {
        let api_key = self.configured_api_key();
        match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
                if let Some(api_key) = api_key {
                    builder = builder.bearer_auth(api_key);
                }
            }
            Backend::Anthropic(_) => {
                builder = builder.header("anthropic-version", ANTHROPIC_VERSION);
                if let Some(api_key) = api_key {
                    builder = builder.header("x-api-key", api_key);
                }
            }
        }
        builder
    }

    pub(crate) fn is_forwarding_auth(&self) -> bool {
        self.config().forward_auth
    }

    pub(crate) fn is_provider_owned_header(&self, name: &str) -> bool {
        match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
                ["authorization", "chatgpt-account-id", "x-openai-fedramp"]
                    .iter()
                    .any(|owned| name.eq_ignore_ascii_case(owned))
            }
            Backend::Anthropic(_) => [
                "authorization",
                "x-api-key",
                "anthropic-beta",
                "anthropic-version",
            ]
            .iter()
            .any(|owned| name.eq_ignore_ascii_case(owned)),
        }
    }

    /// Applies only the caller credential accepted by this provider.
    pub(crate) fn apply_forwarded_auth(
        &self,
        mut builder: RequestBuilder,
        metadata: Option<&Metadata>,
    ) -> RequestBuilder {
        if !self.is_forwarding_auth() {
            return builder;
        }
        let Some(headers) = metadata.and_then(|metadata| metadata.http_headers.as_ref()) else {
            return builder;
        };
        match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => {
                for name in ["authorization", "chatgpt-account-id", "x-openai-fedramp"] {
                    if let Some(value) = headers.get(name) {
                        builder = builder.header(name, sensitive_header(value));
                    }
                }
            }
            Backend::Anthropic(_) => {
                for name in ["authorization", "x-api-key"] {
                    if let Some(value) = headers.get(name) {
                        builder = builder.header(name, sensitive_header(value));
                    }
                }
                if let Some(value) = headers.get("anthropic-beta")
                    && let Some(value) = oauth_beta_header(value)
                {
                    builder = builder.header("anthropic-beta", value);
                }
            }
        }
        builder
    }

    /// Custom per-backend headers to forward on every call.
    pub fn extra_headers(&self) -> &BTreeMap<String, String> {
        &self.config().extra_headers
    }

    /// Default top-level fields to merge into outbound request bodies.
    pub fn extra_body(&self) -> &BTreeMap<String, Value> {
        &self.config().extra_body
    }

    /// Top-level fields dropped from outbound request bodies before `extra_body` is merged.
    pub fn omit_body_fields(&self) -> &BTreeSet<String> {
        &self.config().omit_body_fields
    }

    /// Reasoning effort forced on outbound requests, if the target configures one.
    pub fn reasoning_effort(&self) -> Option<&str> {
        self.config().reasoning_effort.as_deref()
    }

    /// Additional attempts allowed after the initial request.
    pub fn max_retries(&self) -> u32 {
        self.config().max_retries
    }

    /// Cooldown after an exhausted transient completion failure.
    pub fn failure_cooldown(&self) -> Duration {
        self.config().failure_cooldown
    }

    /// Deadline for all attempts and the complete response; `None` leaves the wait unbounded.
    pub fn timeout(&self) -> Option<Duration> {
        self.config().timeout
    }

    /// Whether this backend speaks the Anthropic Messages wire format — the only
    /// one with a `count_tokens` endpoint.
    pub fn is_anthropic(&self) -> bool {
        matches!(self, Backend::Anthropic(_))
    }

    /// The upstream `/v1/messages/count_tokens` URL, derived from the same base
    /// URL join as [`url`](Self::url).
    pub fn count_tokens_url(&self) -> String {
        let base_url = &self.config().base_url;
        anthropic_url(base_url, "/count_tokens").unwrap_or_else(|error| {
            tracing::error!(%error, "Unable to build Anthropic token-counting URL");
            base_url.clone()
        })
    }

    /// Whether an upstream 400 `body` looks like a context-window overflow for
    /// this backend's provider.
    pub(crate) fn is_context_overflow(&self, body: &str) -> bool {
        match self {
            Backend::OpenAiChat(_) | Backend::OpenAiResponses(_) => is_overflow_body(
                body,
                |value| {
                    let error = value.get("error");
                    let field = |name| {
                        error
                            .and_then(|err| err.get(name))
                            .and_then(serde_json::Value::as_str)
                    };
                    // llama-server puts the HTTP status in `code` as a number and
                    // names the reason in `type`, so the canonical code never
                    // matches. Its two overflow messages are worded unlike any
                    // other provider's, which leaves the type as the signal that
                    // holds across releases.
                    field("code") == Some("context_length_exceeded")
                        || field("type") == Some("exceed_context_size_error")
                },
                OPENAI_OVERFLOW_PHRASES,
            ),
            Backend::Anthropic(_) => is_overflow_body(body, |_| false, ANTHROPIC_OVERFLOW_PHRASES),
        }
    }
}

// Retains OAuth markers while keeping provider feature betas backend-owned.
fn sensitive_header(value: &HeaderValue) -> HeaderValue {
    let mut value = value.clone();
    value.set_sensitive(true);
    value
}

fn oauth_beta_header(value: &HeaderValue) -> Option<HeaderValue> {
    let oauth_betas = value
        .to_str()
        .ok()?
        .split(',')
        .map(str::trim)
        .filter(|beta| {
            beta.get(..6)
                .is_some_and(|prefix| prefix.eq_ignore_ascii_case("oauth-"))
        });
    let value = oauth_betas.collect::<Vec<_>>().join(",");
    if value.is_empty() {
        return None;
    }
    let mut value = HeaderValue::from_str(&value).ok()?;
    value.set_sensitive(true);
    Some(value)
}

// Accept either a root `/v1` URL or an already-specific OpenAI endpoint URL.
pub(crate) fn openai_url(base_url: &str, suffix: &str) -> Result<String> {
    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
        message: format!("Invalid OpenAI base URL: {error}"),
    })?;
    let base_path = url.path().trim_end_matches('/');
    let base_root = base_path
        .strip_suffix("/chat/completions")
        .or_else(|| base_path.strip_suffix("/responses"))
        .unwrap_or(base_path);
    url.set_path(&format!("{base_root}{suffix}"));
    Ok(url.into())
}

// Accept a bare host, a `/v1` root, or an already-specific `/v1/messages` URL.
fn anthropic_url(base_url: &str, suffix: &str) -> Result<String> {
    let mut url = reqwest::Url::parse(base_url).map_err(|error| LlmClientError::Configuration {
        message: format!("Invalid Anthropic base URL: {error}"),
    })?;
    let base_path = url.path().trim_end_matches('/');
    let messages_path = if base_path.ends_with("/v1/messages") {
        base_path.to_string()
    } else if base_path.ends_with("/v1") {
        format!("{base_path}/messages")
    } else {
        format!("{base_path}/v1/messages")
    };
    url.set_path(&format!("{messages_path}{suffix}"));
    Ok(url.into())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn config(base_url: &str) -> HttpBackendConfig {
        HttpBackendConfig {
            base_url: base_url.to_string(),
            api_key: Some("secret".to_string()),
            forward_auth: false,
            extra_headers: BTreeMap::new(),
            extra_body: BTreeMap::new(),
            omit_body_fields: BTreeSet::new(),
            reasoning_effort: None,
            max_retries: 0,
            failure_cooldown: Duration::ZERO,
            timeout: None,
        }
    }

    #[test]
    fn openai_chat_url_joins_bare_v1() {
        let backend = Backend::OpenAiChat(config("https://api.openai.com/v1"));
        assert_eq!(backend.url(), "https://api.openai.com/v1/chat/completions");
    }

    #[test]
    fn openai_chat_url_tolerates_trailing_slash_and_existing_suffix() {
        assert_eq!(
            Backend::OpenAiChat(config("https://api.openai.com/v1/")).url(),
            "https://api.openai.com/v1/chat/completions"
        );
        assert_eq!(
            Backend::OpenAiChat(config("https://api.openai.com/v1/chat/completions")).url(),
            "https://api.openai.com/v1/chat/completions"
        );
    }

    #[test]
    fn openai_responses_url_uses_responses_path() {
        assert_eq!(
            Backend::OpenAiResponses(config("https://api.openai.com/v1")).url(),
            "https://api.openai.com/v1/responses"
        );
    }

    #[test]
    fn anthropic_url_join_cases() {
        assert_eq!(
            Backend::Anthropic(config("https://api.anthropic.com")).url(),
            "https://api.anthropic.com/v1/messages"
        );
        assert_eq!(
            Backend::Anthropic(config("https://api.anthropic.com/v1")).url(),
            "https://api.anthropic.com/v1/messages"
        );
        assert_eq!(
            Backend::Anthropic(config("https://api.anthropic.com/v1/messages")).url(),
            "https://api.anthropic.com/v1/messages"
        );
    }

    #[test]
    fn wire_format_matches_variant() {
        assert_eq!(
            Backend::OpenAiChat(config("x")).wire_format(),
            WireFormat::OpenAiChat
        );
        assert_eq!(
            Backend::OpenAiResponses(config("x")).wire_format(),
            WireFormat::OpenAiResponses
        );
        assert_eq!(
            Backend::Anthropic(config("x")).wire_format(),
            WireFormat::AnthropicMessages
        );
    }

    // Header validation follows reqwest for both accepted and rejected bytes.
    #[test]
    fn validates_additional_header_bytes() {
        let cases = [
            ("x-display-name", "café", None),
            (
                "bad header",
                "value",
                Some("invalid HTTP header name \"bad header\""),
            ),
            (
                "x-test-header",
                "bad\nvalue",
                Some("invalid HTTP header value for extra_headers entry \"x-test-header\""),
            ),
        ];

        for (name, value, expected) in cases {
            let mut config = config("x");
            config
                .extra_headers
                .insert(name.to_string(), value.to_string());
            let result = Backend::OpenAiChat(config).validate_configured_headers("model");
            match expected {
                Some(expected) => assert!(
                    result.is_err_and(|error| error.to_string().contains(expected)),
                    "expected {expected:?}"
                ),
                None => result.expect("encodable header must pass validation"),
            }
        }
    }

    // Only static credentials that apply_auth would send are validated.
    #[test]
    fn configured_api_key_validation_matches_auth_application() {
        const INVALID_KEY: &str = "canary\nsecret";
        let mut config = config("x");
        config.api_key = Some(INVALID_KEY.to_string());
        let builders: [fn(HttpBackendConfig) -> Backend; 2] =
            [Backend::OpenAiChat, Backend::Anthropic];
        let client = reqwest::Client::new();

        for build_backend in builders {
            let error = build_backend(config.clone())
                .validate_configured_headers("model")
                .expect_err("invalid API key must fail")
                .to_string();
            assert!(
                error.contains("api_key cannot be encoded as an HTTP header"),
                "{error}"
            );
            assert!(!error.contains(INVALID_KEY), "API key leaked in: {error}");

            let mut forwarded = config.clone();
            forwarded.forward_auth = true;
            let backend = build_backend(forwarded);
            backend
                .validate_configured_headers("model")
                .expect("unused API key must not fail validation");
            let request = backend
                .apply_auth(client.get("https://example.test"))
                .build()
                .expect("request");
            assert!(!request.headers().contains_key("authorization"));
            assert!(!request.headers().contains_key("x-api-key"));
        }
    }

    #[test]
    fn openai_detects_canonical_and_wrapped_overflow() {
        let backend = Backend::OpenAiChat(config("x"));
        assert!(
            backend.is_context_overflow(
                r#"{"error":{"code":"context_length_exceeded","message":"x"}}"#
            )
        );
        // NVIDIA/LiteLLM message wrap with no structured code.
        assert!(backend.is_context_overflow(
            r#"{"error":{"message":"the model's context length is only 131072 tokens"}}"#
        ));
        assert!(!backend.is_context_overflow(r#"{"error":{"code":"invalid_api_key"}}"#));
        // Hub GLM (LiteLLM-wrapped): code is "400", detection relies on phrase match.
        assert!(backend.is_context_overflow(
            r#"{"error":{"message":"Input length 877338 exceeds the maximum allowed input length of 639968 tokens","code":"400"}}"#
        ));
        // Native SGLang: top-level error envelope (no `error` key).
        // KV-pool rejection (managers/utils.py) and declared-context rejection
        // (tokenizer_manager.py); both stable across v0.5.15-v0.5.17.
        assert!(backend.is_context_overflow(
            r#"{"object":"error","message":"Input length (700001 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.","type":"BadRequestError","param":null,"code":400}"#
        ));
        assert!(backend.is_context_overflow(
            r#"{"object":"error","message":"The input (12345 tokens) is longer than the model's context length (8192 tokens).","type":"BadRequestError","param":null,"code":400}"#
        ));
    }

    #[test]
    fn openai_detects_llama_server_overflow() {
        let backend = Backend::OpenAiChat(config("x"));
        // llama-server, numeric `code` and the reason in `type`. Both of the
        // messages ERROR_TYPE_EXCEED_CONTEXT_SIZE can carry are here.
        assert!(backend.is_context_overflow(
            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it","type":"exceed_context_size_error"}}"#
        ));
        assert!(backend.is_context_overflow(
            r#"{"error":{"code":400,"message":"input (600000 tokens) is larger than the max context size (524288 tokens). skipping","type":"exceed_context_size_error"}}"#
        ));
        // The type alone is enough, so a reworded message still falls back.
        assert!(backend.is_context_overflow(
            r#"{"error":{"code":400,"message":"something new","type":"exceed_context_size_error"}}"#
        ));
        // And the message alone is enough, for a proxy that drops the type.
        assert!(backend.is_context_overflow(
            r#"{"error":{"code":400,"message":"request (600000 tokens) exceeds the available context size (524288 tokens), try increasing it"}}"#
        ));
        // llama-server's other 400s must still surface to the caller.
        assert!(!backend.is_context_overflow(
            r#"{"error":{"code":400,"message":"Invalid tool_choice","type":"invalid_request_error"}}"#
        ));
    }

    #[test]
    fn anthropic_detects_prompt_too_long() {
        let backend = Backend::Anthropic(config("x"));
        assert!(
            backend.is_context_overflow(
                r#"{"error":{"message":"prompt is too long: 200000 tokens"}}"#
            )
        );
        assert!(!backend.is_context_overflow(r#"{"error":{"message":"overloaded"}}"#));
    }
}