Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
81.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Rust HTTP server for libsy algorithms.

mod capabilities;
pub mod config;
mod metrics;
mod observability;
mod observed_translation;
mod redaction;
mod response;
mod routing_log;
mod shutdown;
mod sse;
mod stats;
#[cfg(test)]
mod testing;
mod usage_metrics;

use std::collections::BTreeMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
use std::future::Future;
use std::io::IsTerminal;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};

use axum::body::Body;
use axum::extract::rejection::{JsonRejection, QueryRejection};
use axum::extract::{DefaultBodyLimit, Query, Request as HttpRequest, State};
use axum::http::header::CONTENT_TYPE;
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
use libsy::{LibsyError, RoutingOutcome};
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use switchyard_llm_client::{AuxiliaryOperation, RunObservation, RunObserver};
use switchyard_protocol::{LlmClientError, Metadata, ModelId, ProviderExtensions, Request, Usage};
use switchyard_runner::{
    CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,
};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
use tracing::{Instrument, Level};

use switchyard_translation::WireFormat;

use crate::response::into_http_response;
use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};

pub use observability::{flush_observability, initialize_observability};

/// Default TCP listen backlog used by the Rust server.
pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;

/// Default time allowed for active requests to finish during shutdown.
pub const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);

/// Maximum buffered JSON request size accepted by the LLM endpoints.
pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;

const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
const FORWARDED_UPSTREAM_HEADERS: &[&str] = &[
    "baggage",
    "openai-processing-ms",
    // Anthropic spells its correlation id without the `x-` prefix.
    "request-id",
    "traceparent",
    "tracestate",
    "x-litellm-response-cost",
    "x-request-id",
];
const FORWARDED_UPSTREAM_HEADER_PREFIXES: &[&str] =
    &["anthropic-ratelimit-", "x-ratelimit-", "x-upstream-"];
const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;

/// Whether an upstream header is safe and useful to expose downstream.
fn should_forward_upstream_header(name: &HeaderName) -> bool {
    let name = name.as_str();
    FORWARDED_UPSTREAM_HEADERS.contains(&name)
        || FORWARDED_UPSTREAM_HEADER_PREFIXES
            .iter()
            .any(|prefix| name.starts_with(prefix))
}
/// Non-standard status used only in logs and metrics for a request whose
/// downstream client disconnected before any response was written.
const CLIENT_CLOSED_REQUEST: u16 = 499;
const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt");

/// Error returned while configuring or running the server.
#[derive(Debug)]
pub struct ServerError {
    message: String,
}

impl ServerError {
    /// Creates a server error with a user-facing message.
    pub fn new(message: impl Into<String>) -> Self {
        Self {
            message: message.into(),
        }
    }
}

impl Display for ServerError {
    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
        formatter.write_str(&self.message)
    }
}

impl Error for ServerError {}

impl From<RunnerError> for ServerError {
    fn from(error: RunnerError) -> Self {
        let mut message = error.to_string();
        let mut source = error.source();
        while let Some(error) = source {
            message.push_str(": ");
            message.push_str(&error.to_string());
            source = error.source();
        }
        Self::new(message)
    }
}

/// Result returned by server setup and lifecycle operations.
pub type ServerResult<T> = std::result::Result<T, ServerError>;

/// Decision result using the same selected/fallback order as libsy.
#[derive(Serialize)]
struct DecisionResponse {
    selected: DecisionTargetResponse,
    fallbacks: Vec<DecisionTargetResponse>,
    #[serde(skip_serializing_if = "Option::is_none")]
    metadata: Option<libsy::OutcomeMetadata>,
    #[serde(skip_serializing_if = "Option::is_none")]
    response: Option<Value>,
}

/// Response view over one target's existing configuration.
#[derive(Serialize)]
struct DecisionTargetResponse {
    target: String,
    model: ModelId,
    llm_client: DecisionLlmClientResponse,
    extra_body: BTreeMap<String, Value>,
    #[serde(skip_serializing_if = "Option::is_none")]
    reasoning_effort: Option<String>,
}

/// Non-secret client settings needed to call a selected model.
#[derive(Serialize)]
struct DecisionLlmClientResponse {
    format: WireFormat,
    base_url: String,
}

/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
    runner: Arc<Runner>,
    redactor: Arc<redaction::Redactor>,
    fallback_http: reqwest::Client,
    metrics: prometheus::Registry,
    stats: StatsAccumulator,
    routing_log: Option<SharedRoutingLog>,
    track_cache_eligibility: bool,
    model_catalog: Arc<tokio::sync::Mutex<Option<(Instant, Value)>>>,
}

#[derive(Clone)]
struct SharedRoutingLog {
    writer: Arc<Mutex<routing_log::RoutingLog>>,
    path: PathBuf,
}

impl SharedRoutingLog {
    fn new(path: PathBuf) -> ServerResult<Self> {
        Ok(Self {
            writer: Arc::new(Mutex::new(routing_log::RoutingLog::new(path.clone())?)),
            path,
        })
    }

    fn append(
        &self,
        context: routing_log::RoutingLogContext,
        target: &str,
        model: &str,
        tier: Option<&str>,
        usage: &Usage,
    ) {
        if let Err(error) = self
            .writer
            .lock()
            .append(context, target, model, tier, usage)
        {
            tracing::warn!(path = %self.path.display(), %error, "routing log append failed");
        }
    }

    fn snapshot_session(
        &self,
        session_id: &str,
    ) -> std::io::Result<Option<routing_log::SessionStatsSnapshot>> {
        routing_log::snapshot(&self.path, session_id)
    }

    fn append_outcome(
        &self,
        context: &routing_log::RoutingLogContext,
        metadata: libsy::OutcomeMetadata,
    ) {
        if metadata.judgments.is_empty() {
            return;
        }
        if let Err(error) = self.writer.lock().append_outcome(context, metadata) {
            tracing::warn!(path = %self.path.display(), %error, "routing outcome append failed");
        }
    }
}

impl ServerState {
    /// Creates HTTP-server state around an already configured runner.
    pub fn from_runner(runner: Runner) -> ServerResult<Self> {
        let metrics = metrics::registry().map_err(ServerError::new)?;
        let fallback_http = reqwest::Client::builder()
            .redirect(reqwest::redirect::Policy::none())
            .build()
            .map_err(|error| ServerError::new(error.to_string()))?;
        let stats = StatsAccumulator::new(
            metrics.clone(),
            runner.models().map(|model| model.algorithm),
        );
        let redactor = redaction::Redactor::new(runner.provider_api_keys());
        Ok(Self {
            redactor: Arc::new(redactor),
            runner: Arc::new(runner),
            fallback_http,
            metrics,
            stats,
            routing_log: None,
            track_cache_eligibility: tracking_enabled_from_env(),
            model_catalog: Arc::default(),
        })
    }

    /// Enables durable per-request routing records at `path`.
    pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
        self.routing_log = Some(SharedRoutingLog::new(path.into())?);
        Ok(self)
    }

    /// This method loads rates in USD per million tokens for saved routing records.
    pub fn with_pricing_file(self, path: impl AsRef<std::path::Path>) -> ServerResult<Self> {
        let prices = switchyard_costs::Pricing::load(path.as_ref()).map_err(ServerError::new)?;
        let log = self
            .routing_log
            .as_ref()
            .ok_or_else(|| ServerError::new("pricing requires a routing log"))?;
        log.writer.lock().pricing = prices;
        Ok(self)
    }

    /// Returns the route model IDs served by the configured algorithms.
    pub fn models(&self) -> impl Iterator<Item = &str> {
        self.runner.models().map(|model| model.id.as_str())
    }

    /// Returns the caller credential family used by `model`, if any.
    pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> {
        self.runner
            .route(model)
            .and_then(|route| route.caller_auth())
            .map(|kind| kind.as_str())
    }

    fn route_for_model(&self, model: &str) -> Option<&Route> {
        self.runner.route(model)
    }

    fn decision_response(
        &self,
        route_model: &ModelId,
        outcome: &RoutingOutcome,
        response: Option<Value>,
    ) -> Option<DecisionResponse> {
        let description = self.runner.describe_decision(route_model, outcome)?;
        let convert = |target: DecisionTarget| DecisionTargetResponse {
            target: target.target,
            model: target.model,
            llm_client: DecisionLlmClientResponse {
                format: target.format,
                base_url: target.base_url,
            },
            extra_body: target.extra_body,
            reasoning_effort: target.reasoning_effort,
        };
        Some(DecisionResponse {
            selected: convert(description.selected),
            fallbacks: description.fallbacks.into_iter().map(convert).collect(),
            metadata: description.metadata,
            response,
        })
    }
}
/// Runtime options shared by server entry points.
#[derive(Clone, Debug)]
pub struct ServerRunOptions {
    /// Socket address to bind.
    pub addr: SocketAddr,
    /// TCP listen backlog.
    pub backlog: u32,
    /// Validate runtime construction without binding a socket.
    pub dry_run: bool,
    /// Maximum time active requests may drain after shutdown begins.
    pub shutdown_timeout: Duration,
    /// TLS certificate configuration, when HTTPS is enabled.
    pub tls: Option<TlsOptions>,
}

/// TLS certificate paths used by the server.
#[derive(Clone, Debug)]
pub struct TlsOptions {
    /// TLS certificate path in PEM format.
    pub cert: PathBuf,
    /// TLS private-key path in PEM format.
    pub key: PathBuf,
}

impl ServerRunOptions {
    fn is_tls(&self) -> bool {
        self.tls.is_some()
    }
}

/// Validates the runtime and starts the HTTP server unless `dry_run` is set.
pub async fn run_server(state: ServerState, options: ServerRunOptions) -> ServerResult<()> {
    if options.dry_run {
        println!("{}", dry_run_summary(&state));
        return Ok(());
    }

    let server = BoundServer::bind(state, options)?;
    println!("{}", server.startup_banner(std::io::stdout().is_terminal()));
    server.serve(shutdown::signal()).await
}

/// A configured server with its listening socket already bound.
pub struct BoundServer {
    listener: TcpListener,
    router: Router,
    options: ServerRunOptions,
    state: ServerState,
}

impl BoundServer {
    /// Binds the configured address and prepares the HTTP router.
    pub fn bind(state: ServerState, options: ServerRunOptions) -> ServerResult<Self> {
        let listener = bind_tcp_listener(options.addr, options.backlog)?;
        let addr = listener.local_addr().map_err(server_io_error)?;
        Ok(Self {
            listener,
            router: build_switchyard_router(state.clone()),
            options: ServerRunOptions { addr, ..options },
            state,
        })
    }

    /// Returns the actual bound address, including an OS-selected port.
    pub fn local_addr(&self) -> SocketAddr {
        self.options.addr
    }

    /// Serves requests until the supplied shutdown future resolves.
    pub async fn serve(
        self,
        shutdown: impl Future<Output = ()> + Send + 'static,
    ) -> ServerResult<()> {
        let shutdown_timeout = self.options.shutdown_timeout;
        if let Some(tls) = self.options.tls {
            serve_tls(self.listener, self.router, tls, shutdown_timeout, shutdown).await
        } else {
            serve(self.listener, self.router, shutdown_timeout, shutdown).await
        }
    }

    fn startup_banner(&self, color: bool) -> String {
        startup_banner(&self.options, &self.state, color)
    }
}

async fn serve_tls(
    listener: TcpListener,
    router: Router,
    tls: TlsOptions,
    shutdown_timeout: Duration,
    shutdown: impl Future<Output = ()> + Send + 'static,
) -> ServerResult<()> {
    if let Err(error) = rustls::crypto::aws_lc_rs::default_provider().install_default() {
        tracing::debug!(?error, "TLS crypto provider was already installed");
    }

    let config = RustlsConfig::from_pem_file(tls.cert, tls.key)
        .await
        .map_err(server_io_error)?;
    let std_listener = listener.into_std().map_err(server_io_error)?;
    let server = axum_server::from_tcp_rustls(std_listener, config).map_err(server_io_error)?;
    let handle = axum_server::Handle::new();
    let server = server
        .handle(handle.clone())
        .serve(router.into_make_service());
    serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
}

async fn serve(
    listener: TcpListener,
    router: Router,
    shutdown_timeout: Duration,
    shutdown: impl Future<Output = ()> + Send + 'static,
) -> ServerResult<()> {
    let std_listener = listener.into_std().map_err(server_io_error)?;
    let server = axum_server::from_tcp(std_listener).map_err(server_io_error)?;
    let handle = axum_server::Handle::new();
    let server = server
        .handle(handle.clone())
        .serve(router.into_make_service());
    serve_until_shutdown(server, handle, shutdown_timeout, shutdown).await
}

/// Runs the server until it exits or shutdown begins, then drains active requests.
async fn serve_until_shutdown(
    server: impl Future<Output = std::io::Result<()>>,
    handle: axum_server::Handle<SocketAddr>,
    timeout: Duration,
    shutdown: impl Future<Output = ()> + Send + 'static,
) -> ServerResult<()> {
    tokio::pin!(server);
    tokio::select! {
        result = &mut server => result.map_err(server_io_error),
        _ = shutdown => {
            tracing::info!(
                ?timeout,
                "shutdown signal received; draining active requests"
            );
            handle.graceful_shutdown(Some(timeout));
            server.await.map_err(server_io_error)
        }
    }
}

/// Ingress timestamp for one request, taken before any body is read.
#[derive(Clone, Copy)]
struct RequestStart(Instant);

/// Routing-log marker distinguishing classifier and judge calls from answer calls.
const CLASSIFIER_TIER: &str = "classifier";

/// This observer updates backend stats and records classifier and judge calls.
///
/// The routing log also records failed answer calls with unknown cost. Successful
/// answer calls are recorded by [`usage_metrics::observe`] with their terminal usage;
/// recording them here would count each answer twice.
fn stats_observer(
    stats: StatsAccumulator,
    classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,
    route: Route,
) -> RunObserver {
    Arc::new(move |observation| match observation {
        RunObservation::Outcome(metadata) => {
            if let Some((log, context)) = classifier_log.as_ref() {
                log.append_outcome(context, metadata);
            }
        }
        RunObservation::AnswerCall(call) => {
            let latency_ms = call.duration.as_secs_f64() * 1_000.0;
            if call.is_success {
                stats.record_success(&call.selected_model, latency_ms);
            } else {
                if let Some((log, context)) = classifier_log.as_ref() {
                    log.append(
                        context.clone().with_reasoning_effort(
                            route
                                .target(&call.selected_model)
                                .and_then(|target| target.reasoning_effort.as_deref()),
                        ),
                        &call.selected_model,
                        route.provider_model(&call.selected_model),
                        None,
                        &Usage::default(),
                    );
                }
                stats.record_error(&call.selected_model);
            }
        }
        RunObservation::LlmCall(call) | RunObservation::DecisionCall(call) => {
            let latency_ms = call.duration.as_secs_f64() * 1_000.0;
            if let Some((log, context)) = classifier_log.as_ref() {
                let model = route.provider_model(&call.selected_model);
                log.append(
                    context.clone().with_reasoning_effort(
                        route
                            .target(&call.selected_model)
                            .and_then(|target| target.reasoning_effort.as_deref()),
                    ),
                    &call.selected_model,
                    model,
                    Some(CLASSIFIER_TIER),
                    call.usage.as_ref().unwrap_or(&Usage::default()),
                );
            }
            if call.is_success {
                stats.record_classifier_success(
                    call.selected_model,
                    call.usage.as_ref().map(usage_metrics::token_usage),
                    latency_ms,
                );
            } else {
                stats.record_classifier_error(call.selected_model);
            }
        }
        RunObservation::RoutingOverhead(duration) => {
            stats.record_routing_overhead(duration.as_secs_f64() * 1_000.0);
        }
    })
}

/// Stamps the ingress instant into request extensions. Runs as a router layer,
/// so it executes before the handlers' `Json` extractor buffers the body —
/// request-latency measurements therefore include body read and decode.
async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {
    request
        .extensions_mut()
        .insert(RequestStart(Instant::now()));
    next.run(request).await
}

/// Builds an Axum router containing only the primary LLM inference endpoints.
///
/// The registered paths are `/v1/chat/completions`, `/v1/messages`, and `/v1/responses`.
/// The router excludes operational, discovery, auxiliary, and fallback proxy routes so an
/// embedding application can mount those capabilities under its own policies.
pub fn build_llm_router(state: ServerState) -> Router {
    finish_router(primary_llm_routes(), state)
}

/// Builds the full Axum router used by the standalone Switchyard server.
pub fn build_switchyard_router(state: ServerState) -> Router {
    let mut router = primary_llm_routes()
        .route("/v1/decision", post(decision))
        .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
        .route(
            "/v1/responses/input_tokens",
            post(openai_responses_input_tokens),
        )
        .route("/v1/responses/compact", post(openai_responses_compact))
        .route("/v1/models", get(models))
        .route("/v1/stats", get(get_stats))
        .route("/v1/stats/reset", post(reset_stats))
        .route("/metrics", get(prometheus_metrics))
        .route("/health", get(health));
    if state.routing_log.is_some() {
        router = router.route("/v1/routing/session-stats", get(get_session_stats));
    }
    finish_router(router.fallback(proxy_unmatched), state)
}

// Keeps the embedded and standalone servers on the same primary route definitions.
fn primary_llm_routes() -> Router<ServerState> {
    Router::new()
        .route("/v1/chat/completions", post(openai_chat_completions))
        .route("/v1/messages", post(anthropic_messages))
        .route("/v1/responses", post(openai_responses))
}

// Applies the serving limits and request timing shared by both public router constructors.
fn finish_router(router: Router<ServerState>, state: ServerState) -> Router {
    router
        .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
        .layer(axum::middleware::from_fn_with_state(
            state.clone(),
            redaction::redact_response,
        ))
        // `layer` only wraps routes registered before it, so this stays last.
        .layer(axum::middleware::from_fn(stamp_request_start))
        .with_state(state)
}

fn bind_tcp_listener(addr: SocketAddr, backlog: u32) -> ServerResult<TcpListener> {
    let socket = if addr.is_ipv4() {
        TcpSocket::new_v4()
    } else {
        TcpSocket::new_v6()
    }
    .map_err(server_io_error)?;

    socket.set_reuseaddr(true).map_err(server_io_error)?;
    socket.bind(addr).map_err(server_io_error)?;
    socket.listen(backlog).map_err(server_io_error)
}

fn server_io_error(error: std::io::Error) -> ServerError {
    ServerError::new(error.to_string())
}

async fn openai_chat_completions(
    State(state): State<ServerState>,
    Extension(started): Extension<RequestStart>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await
}

async fn anthropic_messages(
    State(state): State<ServerState>,
    Extension(started): Extension<RequestStart>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await
}

async fn openai_responses(
    State(state): State<ServerState>,
    Extension(started): Extension<RequestStart>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await
}

// Forwards unmatched requests unchanged to the configured API root.
async fn proxy_unmatched(State(state): State<ServerState>, request: HttpRequest) -> Response {
    let Some(base_url) = state.runner.fallback_base_url() else {
        return not_found().await;
    };
    let (mut parts, body) = request.into_parts();
    let path_and_query = parts
        .uri
        .path_and_query()
        .map_or_else(|| parts.uri.path().to_string(), ToString::to_string);
    strip_hop_by_hop_headers(&mut parts.headers);
    parts.headers.remove(axum::http::header::HOST);
    let url = match fallback_url(base_url, &path_and_query) {
        Ok(url) => url,
        Err(error) => {
            return error_response(
                StatusCode::BAD_GATEWAY,
                error,
                "upstream_error",
                "upstream_error",
            );
        }
    };
    let body = reqwest::Body::wrap_stream(body.into_data_stream());
    match state
        .fallback_http
        .request(parts.method, url)
        .headers(parts.headers)
        .body(body)
        .send()
        .await
    {
        Ok(response) => {
            let response: http::Response<reqwest::Body> = response.into();
            let (mut parts, body) = response.into_parts();
            strip_hop_by_hop_headers(&mut parts.headers);
            Response::from_parts(parts, Body::new(body))
        }
        Err(error) => error_response(
            StatusCode::BAD_GATEWAY,
            error.without_url().to_string(),
            "upstream_error",
            "upstream_error",
        ),
    }
}

fn fallback_url(base_url: &str, path_and_query: &str) -> Result<reqwest::Url, String> {
    let mut url = reqwest::Url::parse(base_url.trim())
        .map_err(|error| format!("invalid fallback base URL: {error}"))?;
    let base_query = url.query().map(str::to_owned);
    url.set_query(None);
    url.set_fragment(None);

    let base_path = url.path().trim_end_matches('/');
    let root_path = [
        "/v1/chat/completions",
        "/v1/responses",
        "/v1/messages",
        "/v1",
    ]
    .iter()
    .find_map(|suffix| base_path.strip_suffix(suffix))
    .unwrap_or(base_path);
    let (request_path, request_query) = path_and_query
        .split_once('?')
        .map_or((path_and_query, None), |(path, query)| (path, Some(query)));
    let request_path = request_path.strip_prefix('/').unwrap_or(request_path);
    let target_path = if root_path.is_empty() {
        format!("/{request_path}")
    } else if request_path.is_empty() {
        format!("{root_path}/")
    } else {
        format!("{root_path}/{request_path}")
    };
    url.set_path(&target_path);

    let query = match (base_query, request_query) {
        (Some(base_query), Some(request_query)) => Some(format!("{base_query}&{request_query}")),
        (Some(base_query), None) => Some(base_query),
        (None, Some(request_query)) => Some(request_query.to_owned()),
        (None, None) => None,
    };
    url.set_query(query.as_deref());
    Ok(url)
}

fn strip_hop_by_hop_headers(headers: &mut HeaderMap) {
    let connection_headers = headers
        .get_all(axum::http::header::CONNECTION)
        .iter()
        .filter_map(|value| value.to_str().ok())
        .flat_map(|value| value.split(','))
        .filter_map(|name| HeaderName::from_bytes(name.trim().as_bytes()).ok())
        .collect::<Vec<_>>();
    for name in connection_headers {
        headers.remove(name);
    }
    for name in [
        "connection",
        "keep-alive",
        "proxy-authenticate",
        "proxy-authorization",
        "proxy-connection",
        "te",
        "trailer",
        "transfer-encoding",
        "upgrade",
    ] {
        headers.remove(name);
    }
}

/// One provider request submitted for a routing decision without an answer-model call.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct DecisionEndpointRequest {
    input_format: WireFormat,
    request: Value,
}

/// Selects a target while still allowing the algorithm's classifier and judge calls.
async fn decision(
    State(state): State<ServerState>,
    headers: HeaderMap,
    body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,
) -> Response {
    let span = observability::request_span(&headers);
    decision_inner(state, headers, body).instrument(span).await
}

async fn decision_inner(
    state: ServerState,
    headers: HeaderMap,
    body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,
) -> Response {
    let body = match body {
        Ok(Json(body)) if body.request.is_object() => body,
        Ok(_) => {
            return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object");
        }
        Err(error) => {
            return invalid_body_error(
                error.status(),
                format!("Request body must be valid JSON: {error}"),
            );
        }
    };
    let input_format = body.input_format;
    let (route, request) = match resolve_route(
        &state,
        metadata_from_headers(headers),
        body.request,
        input_format,
    ) {
        Ok(resolved) => resolved,
        Err(response) => return response,
    };
    let route_model = request
        .llm_request
        .model
        .as_deref()
        .map(ModelId::from)
        .unwrap_or_default();

    let mut outcome = match route.decide(request).await {
        Ok(outcome) => outcome,
        Err(error) => return runner_error(error),
    };
    let response = match outcome.response.take() {
        Some(response) => {
            let aggregate = match response.llm_response.into_agg().await {
                Ok(aggregate) => aggregate,
                Err(error) => return client_error(&error),
            };
            // The request moved into the decision run, so its namespace mapping
            // is gone by here. A Codex tool call in this preview keeps its
            // qualified name.
            match observed_translation::encode_response(
                &aggregate,
                input_format,
                outcome
                    .selected_model_id()
                    .ok()
                    .map(|model| route.provider_model(model).as_str()),
                &ProviderExtensions::default(),
            ) {
                Ok(response) => Some(response),
                Err(error) => return server_error(error.to_string()),
            }
        }
        None => None,
    };
    match state.decision_response(&route_model, &outcome, response) {
        Some(response) => Json(response).into_response(),
        None => {
            server_error("routing outcome contains a model with no callable target configuration")
        }
    }
}

/// Anthropic token counting against the route's explicitly configured target.
async fn anthropic_count_tokens(
    State(state): State<ServerState>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    let body = match llm_json_body(body) {
        Ok(body) => body,
        Err((status, message)) => {
            return anthropic_error_response(invalid_body_error(status, message));
        }
    };
    let (route, request) = match resolve_route(
        &state,
        metadata_from_headers(headers),
        body,
        WireFormat::AnthropicMessages,
    ) {
        Ok(resolved) => resolved,
        Err(response) => return anthropic_error_response(response),
    };
    anthropic_error_response(
        match route
            .call_auxiliary(request, AuxiliaryOperation::AnthropicCountTokens)
            .await
        {
            Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
            Err(RunnerError::AuxiliaryUnsupported) => error_response(
                StatusCode::BAD_REQUEST,
                "route has no Anthropic target for token counting",
                "invalid_request_error",
                "count_tokens_unsupported",
            ),
            Err(RunnerError::Client(error)) => client_error(&error),
            Err(error) => server_error(error.to_string()),
        },
    )
}

async fn openai_responses_input_tokens(
    State(state): State<ServerState>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    openai_responses_auxiliary(
        state,
        headers,
        body,
        AuxiliaryOperation::ResponsesInputTokens,
    )
    .await
}

async fn openai_responses_compact(
    State(state): State<ServerState>,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
    openai_responses_auxiliary(state, headers, body, AuxiliaryOperation::ResponsesCompact).await
}

// Resolves route aliases and configured credentials before calling a model-bearing Responses API.
async fn openai_responses_auxiliary(
    state: ServerState,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
    operation: AuxiliaryOperation,
) -> Response {
    let body = match llm_json_body(body) {
        Ok(body) => body,
        Err((status, message)) => {
            return render_error_response(
                invalid_body_error(status, message),
                WireFormat::OpenAiResponses,
            );
        }
    };
    let (route, request) = match resolve_route(
        &state,
        metadata_from_headers(headers),
        body,
        WireFormat::OpenAiResponses,
    ) {
        Ok(resolved) => resolved,
        Err(response) => return render_error_response(response, WireFormat::OpenAiResponses),
    };
    let result = route.call_auxiliary(request, operation).await;
    render_error_response(
        match result {
            Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
            Err(RunnerError::AuxiliaryUnsupported) => error_response(
                StatusCode::BAD_REQUEST,
                "route has no OpenAI Responses target",
                "invalid_request_error",
                "responses_auxiliary_unsupported",
            ),
            Err(RunnerError::Client(error)) => client_error(&error),
            Err(error) => server_error(error.to_string()),
        },
        WireFormat::OpenAiResponses,
    )
}

async fn handle_endpoint(
    state: ServerState,
    started: RequestStart,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
    wire_format: WireFormat,
) -> Response {
    let span = observability::request_span(&headers);
    handle_endpoint_inner(state, started, headers, body, wire_format)
        .instrument(span)
        .await
}

async fn handle_endpoint_inner(
    state: ServerState,
    started: RequestStart,
    headers: HeaderMap,
    body: std::result::Result<Json<Value>, JsonRejection>,
    wire_format: WireFormat,
) -> Response {
    let metadata = metadata_from_headers(headers);
    let routing_log_context = state
        .routing_log
        .as_ref()
        .map(|_| routing_log::RoutingLogContext::from_metadata(&metadata));
    let request_log = RequestLogContext {
        started: started.0,
        wire_format,
        requested_model: body
            .as_ref()
            .ok()
            .and_then(|body| body.0.get("model"))
            .and_then(Value::as_str)
            .map(str::to_string),
        streaming: body
            .as_ref()
            .ok()
            .and_then(|body| body.0.get("stream"))
            .and_then(Value::as_bool)
            .unwrap_or(false),
        session_id: metadata.session_id.clone(),
        correlation_id: metadata.correlation_id.clone(),
    };

    // Axum drops this future when the downstream client disconnects before the
    // response is written, which cancels the run and its upstream calls. The guard
    // keeps that terminal state observable: the `emit` path below never runs for an
    // abandoned buffered request.
    let mut request_log = RequestLogGuard(Some(request_log));

    let response = match llm_json_body(body) {
        Ok(body) => {
            handle_llm_request(
                state,
                started,
                metadata,
                body,
                wire_format,
                routing_log_context,
            )
            .await
        }
        Err((status, message)) => invalid_body_error(status, message),
    };
    let response = render_error_response(response, wire_format);
    metrics::record_client_response(response.status().as_u16());
    request_log.emit(&response);
    response
}

fn llm_json_body(
    body: std::result::Result<Json<Value>, JsonRejection>,
) -> std::result::Result<Value, (StatusCode, String)> {
    match body {
        Ok(Json(value)) if value.is_object() => Ok(value),
        Ok(_) => Err((
            StatusCode::BAD_REQUEST,
            "Request body must be a JSON object".to_string(),
        )),
        Err(error) => Err((
            error.status(),
            format!("Request body must be valid JSON: {error}"),
        )),
    }
}

/// Decode `body`, resolve the route named by its `model`, and build the
/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns
/// the resolved route and the built request — or an error [`Response`]
/// (invalid body, empty `model` → 400, unknown route → 404).
// Both callers immediately return the `Err(Response)` as the HTTP response, so
// the large error type is intentional, not propagated up a call stack.
#[allow(clippy::type_complexity, clippy::result_large_err)]
fn resolve_route(
    state: &ServerState,
    metadata: Metadata,
    mut body: Value,
    wire_format: WireFormat,
) -> std::result::Result<(&Route, Request), Response> {
    // Only trusted translation hops may supply exact request preservation state.
    // Strip it before decoding and retaining the raw body for upstream replay.
    if let Some(metadata) = body.get_mut("metadata").and_then(Value::as_object_mut) {
        metadata.remove(switchyard_translation::util::SWITCHYARD_METADATA_KEY);
    }
    let mut llm_request = observed_translation::decode_request(wire_format, &body)
        .map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
    switchyard_translation::normalize_compaction(&mut llm_request, metadata.http_headers.as_ref());
    let requested_model = llm_request
        .model
        .clone()
        .filter(|model| !model.trim().is_empty())
        .ok_or_else(|| {
            error_response(
                StatusCode::BAD_REQUEST,
                "request body must include a non-empty string `model`",
                "invalid_request_error",
                "invalid_request_error",
            )
        })?;
    let route = state.route_for_model(&requested_model).ok_or_else(|| {
        error_response(
            StatusCode::NOT_FOUND,
            format!("No route registered for model {requested_model}"),
            "model_not_found",
            "model_not_found",
        )
    })?;
    if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) =
        route.check_caller_format(wire_format)
    {
        let (provider, expected_endpoint) = match caller_auth {
            CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"),
            CallerAuthKind::OpenAi => ("OpenAI", "/v1/chat/completions or /v1/responses"),
        };
        return Err(error_response(
            StatusCode::BAD_REQUEST,
            format!(
                "route {requested_model} forwards an {provider} login; call it through {expected_endpoint}",
            ),
            "invalid_request_error",
            "invalid_request_error",
        ));
    }
    if let Some(capability) =
        capabilities::unsupported_capability(route.capabilities(), &llm_request, &body)
    {
        return Err(error_response(
            StatusCode::BAD_REQUEST,
            format!(
                "route {requested_model} declares {capability} = false; remove the unsupported input or select another route"
            ),
            "invalid_request_error",
            "unsupported_capability",
        ));
    }
    let request = Request {
        llm_request,
        raw_request: Some(body),
        metadata: Some(metadata),
    };
    Ok((route, request))
}

/// Resolves and executes an LLM request, attaching route identity when durable logging is enabled.
async fn handle_llm_request(
    state: ServerState,
    started: RequestStart,
    metadata: Metadata,
    body: Value,
    wire_format: WireFormat,
    routing_log_context: Option<routing_log::RoutingLogContext>,
) -> Response {
    let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));
    let (route, request) = match resolve_route(&state, metadata, body, wire_format) {
        Ok(resolved) => resolved,
        Err(response) => return response,
    };
    let routing_log_context = routing_log_context.map(|context| {
        context.with_route(
            request.llm_request.model.as_deref().unwrap_or_default(),
            route.algorithm_name(),
        )
    });
    // Only the Codex namespace mapping is needed downstream, not the whole request.
    let request_extensions = request.llm_request.extensions.clone();
    let stats = state.stats.for_route(route.clone());
    let observer = stats_observer(
        stats.clone(),
        state.routing_log.clone().zip(routing_log_context.clone()),
        route.clone(),
    );

    let output = match route.execute(request, Some(observer)).await {
        Ok(output) => output,
        Err(error) => {
            if let RunnerError::Algorithm(LibsyError::ClientCall {
                target,
                source:
                    LlmClientError::ResponseStateLimitExceeded { .. }
                    | LlmClientError::ResponseStateConflict,
            }) = &error
            {
                stats.record_response_error(target);
            }
            return runner_error(error);
        }
    };
    let RunOutput {
        selected_model,
        response,
    } = output;
    // The response carries the candidate that actually served it. Fall back to the routing
    // selection for algorithms that return a response without an offloaded model call.
    let served_model = response.served_model().cloned().or(Some(selected_model));
    // Keep the provider identity separate from the selected target for reporting.
    let provider_model = served_model
        .as_ref()
        .map(|model| route.provider_model(model).to_string());
    let mut response = if let Some(served_model) = served_model.as_ref() {
        let cache_eligible = cache_probe
            .as_ref()
            .map(|probe| stats.prefix_eligibility(served_model, probe))
            .unwrap_or(0.0);
        let routing_log_context = routing_log_context.map(|context| {
            context.with_reasoning_effort(
                route
                    .target(served_model)
                    .and_then(|target| target.reasoning_effort.as_deref()),
            )
        });
        usage_metrics::observe(
            response,
            served_model.as_str(),
            provider_model.as_deref().unwrap_or(served_model.as_str()),
            started.0,
            stats,
            cache_eligible,
            state.routing_log.zip(routing_log_context),
        )
    } else {
        response
    };

    let upstream_headers = std::mem::take(&mut response.upstream_headers);
    let response_model = provider_model.clone();
    let mut response = match into_http_response(
        response,
        wire_format,
        response_model,
        request_extensions,
        Arc::clone(&state.redactor),
    ) {
        Ok(response) => response,
        Err(error) => return server_error(error.to_string()),
    };
    // Forward upstream headers before Switchyard writes its own so any header
    // this server emits always overrides an upstream echo of the same name.
    let response_headers = response.headers_mut();
    for (name, value) in upstream_headers.iter() {
        if !should_forward_upstream_header(name) {
            continue;
        }
        response_headers.append(name.clone(), value.clone());
    }
    if let Some(provider_model) = provider_model.as_ref() {
        attach_routing_headers(&mut response, provider_model);
    }
    response
}

/// Holds the request log context until the request reaches a terminal state.
/// Dropping the guard with the context still in place means the handler future was
/// cancelled — the client left before a response existed — so the run is reported
/// as abandoned rather than silently disappearing.
struct RequestLogGuard(Option<RequestLogContext>);

impl RequestLogGuard {
    // Emits the normal terminal event and disarms the cancellation path.
    fn emit(&mut self, response: &Response) {
        if let Some(context) = self.0.take() {
            context.emit(response);
        }
    }
}

impl Drop for RequestLogGuard {
    fn drop(&mut self) {
        if let Some(context) = self.0.take() {
            context.emit_cancelled();
        }
    }
}

// Request metadata held until the terminal response determines the event level.
struct RequestLogContext {
    started: Instant,
    wire_format: WireFormat,
    requested_model: Option<String>,
    streaming: bool,
    session_id: Option<String>,
    correlation_id: Option<String>,
}

// Error text carried separately so terminal logging never consumes an HTTP body.
#[derive(Clone)]
struct RequestLogError(String);

impl RequestLogContext {
    fn emit(self, response: &Response) {
        let selected_model = response
            .headers()
            .get(HEADER_SELECTED_MODEL)
            .and_then(|value| value.to_str().ok())
            .unwrap_or("");
        let duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0;
        let error = response
            .extensions()
            .get::<RequestLogError>()
            .map(|error| error.0.as_str())
            .unwrap_or("");

        macro_rules! emit {
            ($level:expr, $message:literal) => {
                tracing::event!(
                    target: "switchyard_server::request",
                    $level,
                    wire_format = %self.wire_format,
                    status = response.status().as_u16(),
                    requested_model = self.requested_model.as_deref().unwrap_or(""),
                    selected_model,
                    streaming = self.streaming,
                    session_id = self.session_id.as_deref(),
                    correlation_id = self.correlation_id.as_deref(),
                    handling_duration_ms = duration_ms,
                    error,
                    $message
                )
            };
        }

        match request_log_level(response.status()) {
            Level::ERROR => emit!(Level::ERROR, "LLM request failed"),
            Level::WARN => emit!(Level::WARN, "LLM request failed"),
            _ => emit!(Level::INFO, "LLM request handled"),
        }
    }

    // Terminal event for a request whose client left before a response was written.
    // No model served it, so there is no selected model and no response status.
    fn emit_cancelled(self) {
        metrics::record_client_disconnect();
        tracing::event!(
            target: "switchyard_server::request",
            Level::WARN,
            wire_format = %self.wire_format,
            status = CLIENT_CLOSED_REQUEST,
            requested_model = self.requested_model.as_deref().unwrap_or(""),
            selected_model = "",
            streaming = self.streaming,
            session_id = self.session_id.as_deref(),
            correlation_id = self.correlation_id.as_deref(),
            handling_duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0,
            error = "client disconnected before a response was written",
            "LLM request cancelled"
        );
    }
}

fn request_log_level(status: StatusCode) -> Level {
    if status.is_server_error() {
        Level::ERROR
    } else if status.is_success() {
        Level::INFO
    } else {
        Level::WARN
    }
}

fn metadata_from_headers(headers: HeaderMap) -> Metadata {
    let mut metadata = Metadata::from_headers(&headers);
    metadata.http_headers = Some(headers);
    metadata
}

fn attach_routing_headers(response: &mut Response, served_model: &str) {
    insert_routing_header(response, HEADER_SELECTED_MODEL, served_model);
}

fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {
    let Some(value) = sanitize_routing_header_value(value) else {
        return;
    };
    let Ok(value) = HeaderValue::from_str(&value) else {
        return;
    };
    response
        .headers_mut()
        .insert(HeaderName::from_static(name), value);
}

fn sanitize_routing_header_value(value: &str) -> Option<String> {
    let value = value.split_whitespace().collect::<Vec<_>>().join(" ");
    (!value.is_empty()).then(|| value.chars().take(MAX_ROUTING_HEADER_VALUE_LEN).collect())
}

fn algorithm_error(error: LibsyError) -> Response {
    let LibsyError::ClientCall { source, .. } = &error else {
        return server_error(error.to_string());
    };
    client_error(source)
}

fn runner_error(error: RunnerError) -> Response {
    match error {
        RunnerError::Algorithm(error) => algorithm_error(error),
        RunnerError::Client(error) => client_error(&error),
        error => server_error(error.to_string()),
    }
}

fn client_error(error: &LlmClientError) -> Response {
    match error {
        LlmClientError::ResponseStateLimitExceeded { .. } => error_response(
            StatusCode::SERVICE_UNAVAILABLE,
            error.to_string(),
            "server_error",
            "response_state_limit_exceeded",
        ),
        LlmClientError::ResponseStateConflict => error_response(
            StatusCode::CONFLICT,
            error.to_string(),
            "server_error",
            "response_state_conflict",
        ),
        LlmClientError::InvalidRequest { message }
        | LlmClientError::RequestTranslation(message) => error_response(
            StatusCode::BAD_REQUEST,
            message,
            "invalid_request_error",
            "invalid_request_error",
        ),
        LlmClientError::Configuration { message } => error_response(
            StatusCode::BAD_GATEWAY,
            message,
            "upstream_error",
            "upstream_configuration_error",
        ),
        LlmClientError::ContextWindowExceeded { message, .. } => error_response(
            StatusCode::BAD_REQUEST,
            message,
            "invalid_request_error",
            "context_length_exceeded",
        ),
        LlmClientError::TemporarilyUnavailable => error_response(
            StatusCode::SERVICE_UNAVAILABLE,
            error.to_string(),
            "upstream_error",
            "temporarily_unavailable",
        ),
        LlmClientError::UpstreamHttp { status, body } => upstream_error(*status, body),
        LlmClientError::Transport { source } | LlmClientError::InvalidResponse { source } => {
            error_response(
                StatusCode::BAD_GATEWAY,
                source.to_string(),
                "upstream_error",
                "upstream_error",
            )
        }
        LlmClientError::ResponseTranslation(message) => error_response(
            StatusCode::BAD_GATEWAY,
            message,
            "upstream_error",
            "upstream_error",
        ),
        LlmClientError::Timeout { source } => error_response(
            StatusCode::GATEWAY_TIMEOUT,
            source.to_string(),
            "upstream_error",
            "upstream_timeout",
        ),
        LlmClientError::RequestEncoding(message) => server_error(message),
        _ => server_error(error.to_string()),
    }
}

// Keep the provider's message and nonempty string code in our error JSON.
fn upstream_error(status: StatusCode, body: &str) -> Response {
    let parsed = serde_json::from_str::<Value>(body).unwrap_or_default();
    let error = &parsed["error"];
    let message = error["message"].as_str().unwrap_or(body);
    let code = error["code"]
        .as_str()
        .filter(|code| !code.is_empty())
        .unwrap_or("upstream_error");
    let mut response = error_response(status, message, "upstream_error", code);
    // Provider messages and codes can quote request content; log only fixed metadata.
    response
        .extensions_mut()
        .insert(RequestLogError(format!("upstream_error (HTTP {status})")));
    response
}

// Keep error details until the endpoint chooses its response format.
#[derive(Clone)]
struct ApiError {
    status: StatusCode,
    message: String,
    error_type: &'static str,
    code: String,
}

impl ApiError {
    fn new(
        status: StatusCode,
        message: impl Into<String>,
        error_type: &'static str,
        code: impl Into<String>,
    ) -> Self {
        Self {
            status,
            message: message.into(),
            error_type,
            code: code.into(),
        }
    }

    fn into_response(self, wire_format: WireFormat) -> Response {
        let body = match wire_format {
            WireFormat::AnthropicMessages => json!({
                "type": "error",
                "error": {
                    "type": anthropic_error_type(self.status),
                    "message": self.message.clone(),
                }
            }),
            WireFormat::OpenAiChat | WireFormat::OpenAiResponses => json!({
                "error": {
                    "message": self.message.clone(),
                    "type": self.error_type,
                    "code": self.code,
                }
            }),
        };
        let mut response = (self.status, Json(body)).into_response();
        response
            .extensions_mut()
            .insert(RequestLogError(self.message.clone()));
        response.extensions_mut().insert(self);
        response
    }
}

fn render_error_response(mut response: Response, wire_format: WireFormat) -> Response {
    let Some(error) = response.extensions_mut().remove::<ApiError>() else {
        return response;
    };
    let log_error = response.extensions_mut().remove::<RequestLogError>();
    let mut rendered = error.into_response(wire_format);
    if let Some(log_error) = log_error {
        rendered.extensions_mut().insert(log_error);
    }
    rendered
}

fn anthropic_error_response(response: Response) -> Response {
    render_error_response(response, WireFormat::AnthropicMessages)
}

fn anthropic_error_type(status: StatusCode) -> &'static str {
    match status {
        StatusCode::BAD_REQUEST => "invalid_request_error",
        StatusCode::UNAUTHORIZED => "authentication_error",
        StatusCode::FORBIDDEN => "permission_error",
        StatusCode::NOT_FOUND => "not_found_error",
        StatusCode::PAYLOAD_TOO_LARGE => "request_too_large",
        StatusCode::TOO_MANY_REQUESTS => "rate_limit_error",
        status if status.as_u16() == 529 => "overloaded_error",
        _ => "api_error",
    }
}

fn server_error(message: impl Into<String>) -> Response {
    error_response(
        StatusCode::INTERNAL_SERVER_ERROR,
        message,
        "server_error",
        "server_error",
    )
}

fn invalid_body_error(status: StatusCode, message: impl Into<String>) -> Response {
    error_response(status, message, "invalid_request_error", "invalid_body")
}

fn error_response(
    status: StatusCode,
    message: impl Into<String>,
    error_type: &'static str,
    code: impl Into<String>,
) -> Response {
    ApiError::new(status, message, error_type, code).into_response(WireFormat::OpenAiChat)
}

async fn models(State(state): State<ServerState>) -> Json<Value> {
    let mut cache = state.model_catalog.lock().await;
    if let Some((updated, catalog)) = cache.as_ref()
        && updated.elapsed() < Duration::from_secs(60)
    {
        return Json(catalog.clone());
    }
    let mut models: Vec<_> = state.runner.models().collect();
    let probes = models
        .iter_mut()
        .filter(|model| model.capabilities.context_window.is_none())
        .map(|model| async {
            model.capabilities.context_window =
                discover_context_window(&state.runner, model.id).await;
        });
    // Keep completed discoveries even when another backend exhausts the shared budget.
    let _ = tokio::time::timeout(
        Duration::from_secs(2),
        futures_util::future::join_all(probes),
    )
    .await;
    let catalog = model_list_payload(
        models
            .into_iter()
            .map(|model| (model.id.as_str(), model.capabilities)),
    );
    *cache = Some((Instant::now(), catalog.clone()));
    Json(catalog)
}

async fn discover_context_window(runner: &Runner, model: &ModelId) -> Option<u32> {
    let body = runner
        .model_properties(model.as_str(), Duration::from_secs(2))
        .await?;
    let context = body
        .pointer("/default_generation_settings/n_ctx")?
        .as_u64()?;
    u32::try_from(context).ok().filter(|context| *context > 0)
}

async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
    Json(state.stats.snapshot())
}

async fn reset_stats(State(state): State<ServerState>) -> Json<Value> {
    state.stats.reset();
    Json(json!({"status": "reset"}))
}

#[derive(Deserialize)]
struct SessionStatsQuery {
    session_id: String,
}

// TODO: This loads the entire file. It should stream the JSONL instead.
// Huge files will crash the demo server.
async fn get_session_stats(
    State(state): State<ServerState>,
    query: std::result::Result<Query<SessionStatsQuery>, QueryRejection>,
) -> Response {
    let Query(query) = match query {
        Ok(query) => query,
        Err(error) => {
            return error_response(
                StatusCode::BAD_REQUEST,
                error.to_string(),
                "invalid_request_error",
                "invalid_query",
            );
        }
    };
    let Some(routing_log) = state.routing_log else {
        // Should be unreachable
        return not_found().await;
    };
    let session_id = query.session_id.clone();
    // Loading and de-serializing a large file is a time consuming blocking operation
    let snapshot =
        match task::spawn_blocking(move || routing_log.snapshot_session(&session_id)).await {
            Ok(s) => s,
            Err(err) => {
                return server_error(format!("failed to snapshot: {err}"));
            }
        };

    match snapshot {
        Ok(Some(snapshot)) => (StatusCode::OK, Json(snapshot)).into_response(),
        Ok(None) => error_response(
            StatusCode::NOT_FOUND,
            "Routing session not found",
            "not_found",
            "routing_session_not_found",
        ),
        Err(error) => server_error(format!("failed to read routing log: {error}")),
    }
}

async fn health() -> Json<Value> {
    Json(json!({"status": "ok"}))
}

async fn prometheus_metrics(State(state): State<ServerState>) -> Response {
    match metrics::encode(&state.metrics) {
        Ok(body) => ([(CONTENT_TYPE, metrics::CONTENT_TYPE)], body).into_response(),
        Err(error) => server_error(error),
    }
}

async fn not_found() -> Response {
    error_response(
        StatusCode::NOT_FOUND,
        "Not Found",
        "not_found",
        "endpoint_not_found",
    )
}

fn model_list_payload<'a>(
    entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
) -> Value {
    let mut entries = entries.into_iter().collect::<Vec<_>>();
    entries.sort_unstable_by_key(|(model_id, _)| *model_id);
    let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
    let first_id = model_ids.first().copied();
    let last_id = model_ids.last().copied();
    json!({
        "object": "list",
        "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
        // Codex requires this key; an empty list preserves its own catalog and instructions.
        "models": [],
        "first_id": first_id,
        "last_id": last_id,
        "has_more": false,
        "default_model": first_id,
        "model_pool": model_ids,
    })
}

fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
    json!({
        "id": model,
        "object": "model",
        "type": "model",
        "created": 0,
        "owned_by": "switchyard",
        "display_name": model,
        // OpenAI-compatible clients read the context window from this field.
        "context_length": capabilities.context_window,
        "capabilities": {
            "streaming": true,
            "tool_calling": capabilities.tool_calling,
            "vision": capabilities.vision,
            "supported_inbound_formats": [
                "openai-chat-completions",
                "openai-responses",
                "anthropic-messages",
            ],
        },
    })
}

fn startup_banner(options: &ServerRunOptions, state: &ServerState, color: bool) -> String {
    let scheme = if options.is_tls() { "https" } else { "http" };
    let listen_url = url_for_addr(scheme, options.addr);
    let request_url = request_url_for_addr(scheme, options.addr);
    let routes = state.models().collect::<Vec<_>>();
    let route_list = routes.join(", ");
    let example_model = routes.first().copied().unwrap_or("switchyard/route");
    let example_body = json!({
        "model": example_model,
        "messages": [{"role": "user", "content": "Hello from Switchyard"}],
    });
    let example_url = shell_quote(&format!("{request_url}/v1/chat/completions"));
    let example_body = shell_quote(&example_body.to_string());
    format!(
        "{}\nSwitchyard libsy server\n  listening: {}\n  routes: {}\n\nendpoints:\n{}\n\nexample:\n  curl -s {} \\\n    -H 'Content-Type: application/json' \\\n    -d {}",
        render_startup_banner_art(color),
        listen_url,
        route_list,
        endpoint_listing(state.routing_log.is_some()),
        example_url,
        example_body,
    )
}

// Keep redirected logs plain. Terminal output applies NVIDIA-green ANSI truecolor per line.
fn render_startup_banner_art(color: bool) -> String {
    let banner = STARTUP_BANNER_ART.trim_end();
    if !color {
        return banner.to_string();
    }

    let (red, green, blue) = (118, 185, 0);
    let mut rendered = String::new();
    for line in banner.lines() {
        rendered.push_str(&format!("\x1b[38;2;{red};{green};{blue}m{line}\x1b[0m\n"));
    }
    rendered.trim_end_matches('\n').to_string()
}

fn dry_run_summary(state: &ServerState) -> String {
    format!(
        "server OK: {}",
        state.models().collect::<Vec<_>>().join(", ")
    )
}

fn url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
    format!("{scheme}://{}:{}", host_for_url(addr.ip()), addr.port())
}

// Use loopback in request examples when the listener binds all interfaces.
fn request_url_for_addr(scheme: &'static str, addr: SocketAddr) -> String {
    let host = if addr.ip().is_unspecified() {
        match addr.ip() {
            std::net::IpAddr::V4(_) => "127.0.0.1".to_string(),
            std::net::IpAddr::V6(_) => "[::1]".to_string(),
        }
    } else {
        host_for_url(addr.ip())
    };
    format!("{scheme}://{host}:{}", addr.port())
}

// Bracket IPv6 literals so they are valid inside a URL authority.
fn host_for_url(ip: std::net::IpAddr) -> String {
    match ip {
        std::net::IpAddr::V4(ip) => ip.to_string(),
        std::net::IpAddr::V6(ip) => format!("[{ip}]"),
    }
}

fn shell_quote(value: &str) -> String {
    format!("'{}'", value.replace('\'', "'\\''"))
}

fn endpoint_listing(has_routing_log: bool) -> String {
    let mut endpoints = vec![
        "  POST /v1/chat/completions    OpenAI Chat Completions",
        "  POST /v1/messages            Anthropic Messages",
        "  POST /v1/responses           OpenAI Responses",
        "  POST /v1/messages/count_tokens",
        "  POST /v1/responses/input_tokens",
        "  POST /v1/responses/compact",
        "  ANY  unmatched paths          optional fallback client",
        "  GET  /v1/models              configured routes",
        "  GET  /v1/stats               routing stats",
        "  POST /v1/stats/reset",
        "  GET  /metrics                Prometheus metrics",
        "  GET  /health",
    ];
    if has_routing_log {
        endpoints.push("  GET  /v1/routing/session-stats");
    }
    endpoints.join("\n")
}

#[cfg(test)]
mod tests {
    use switchyard_llm_client::ModelCallObservation;
    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use tokio::sync::{Notify, oneshot};

    use super::*;

    #[tokio::test]
    async fn decision_continues_incoming_w3c_trace_context() {
        use std::io::Write as _;

        use opentelemetry::trace::TracerProvider as _;
        use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
        use tower::ServiceExt as _;
        use tracing_subscriber::layer::SubscriberExt as _;

        let exporter = InMemorySpanExporter::default();
        let provider = SdkTracerProvider::builder()
            .with_simple_exporter(exporter.clone())
            .build();
        let subscriber = tracing_subscriber::registry()
            .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("decision-test")));
        let _guard = tracing::subscriber::set_default(subscriber);

        let mut config = tempfile::NamedTempFile::new().expect("config file");
        config
            .write_all(
                br#"schema_version = 1

[llm_clients.upstream]
format = "openai_chat"
base_url = "http://127.0.0.1:9/v1"

[targets.only]
id = "only-model"
llm_client = "upstream"

[routes.pick]
id = "pick"
type = "random"
targets = ["only"]
"#,
            )
            .expect("write config");
        let app = build_switchyard_router(config::load_server_state(config.path()).expect("state"));

        let request = axum::http::Request::post("/v1/decision")
            .header("content-type", "application/json")
            .header(
                "traceparent",
                "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
            )
            .body(axum::body::Body::from(
                r#"{"input_format":"openai_chat","request":{"model":"pick","messages":[{"role":"user","content":"hi"}]}}"#,
            ))
            .expect("request");
        let response = app.oneshot(request).await.expect("response");
        assert_eq!(response.status(), StatusCode::OK);

        let spans = exporter.get_finished_spans().expect("finished spans");
        let span = |name: &str| {
            spans
                .iter()
                .find(|span| span.name == name)
                .unwrap_or_else(|| panic!("{name} span"))
        };
        let request_span = span("switchyard.request");
        assert_eq!(
            request_span.span_context.trace_id().to_string(),
            "4bf92f3577b34da6a3ce929d0e0e4736"
        );
        assert_eq!(request_span.parent_span_id.to_string(), "00f067aa0ba902b7");
        let run_span = span("libsy.run");
        assert_eq!(
            run_span.span_context.trace_id(),
            request_span.span_context.trace_id()
        );
    }

    /// The observer logs judge calls with their model IDs, classifier tier, and
    /// recorded session and turn IDs. The served response logs routed calls with
    /// their final usage, so logging them here would count each routed call twice.
    #[test]
    fn stats_observer_logs_judge_calls_to_the_routing_log() -> Result<(), Box<dyn std::error::Error>>
    {
        let dir = tempfile::tempdir().expect("temp dir");
        let log = SharedRoutingLog::new(dir.path().join("routing.jsonl")).expect("routing log");
        let mut headers = HeaderMap::new();
        headers.insert("proxy_x_session_id", "session-1".parse().expect("header"));
        headers.insert("x-switchyard-turn-id", "turn-1".parse()?);
        let metadata = metadata_from_headers(headers);
        let context = routing_log::RoutingLogContext::from_metadata(&metadata);
        let runner = Runner::from_toml(
            r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "http://localhost:1/v1"
[targets.judge]
id = "judge-model"
llm_client = "upstream"
[routes.only]
id = "route"
type = "passthrough"
target = "judge"
"#,
        )?;
        let observer = stats_observer(
            StatsAccumulator::default(),
            Some((log.clone(), context)),
            runner.route("route").expect("configured route").clone(),
        );

        let call = |model: &str, answer: bool| {
            let observation = ModelCallObservation {
                selected_model: ModelId::from(model),
                is_success: true,
                duration: Duration::from_millis(3),
                usage: Some(Usage {
                    input_tokens: Some(100),
                    output_tokens: Some(7),
                    ..Usage::default()
                }),
            };
            if answer {
                RunObservation::AnswerCall(observation)
            } else {
                RunObservation::LlmCall(observation)
            }
        };
        observer(call("judge", false));
        observer(call("routed-model", true));
        observer(RunObservation::Outcome(libsy::OutcomeMetadata::new(
            "stage".into(),
            None,
        )));
        for (target, category) in [
            ("strong", switchyard_protocol::Category::Capable),
            ("weak", switchyard_protocol::Category::Efficient),
        ] {
            let mut outcome = libsy::OutcomeMetadata::new(
                "composite".into(),
                Some(serde_json::json!({"source": "stage"})),
            );
            outcome.judgments.push(libsy::JudgeDecision {
                target: Some(target.into()),
                category: Some(category.as_str().to_string()),
                evidence: None,
            });
            observer(RunObservation::Outcome(outcome));
        }

        let snapshot = log
            .snapshot_session("session-1")
            .expect("read log")
            .expect("session recorded");
        let snapshot = serde_json::to_value(&snapshot).expect("serializable snapshot");
        assert_eq!(snapshot["models"]["judge-model"]["calls"], 1);
        assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);
        assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);
        assert!(snapshot["models"].get("routed-model").is_none());
        assert_eq!(snapshot["total_calls"], 1);
        assert_eq!(snapshot["outcomes"].as_array().expect("outcomes").len(), 2);
        assert_eq!(snapshot["outcomes"][0]["judgments"][0]["target"], "strong");
        assert_eq!(
            snapshot["outcomes"][1]["judgments"][0]["category"],
            "efficient"
        );
        assert_eq!(snapshot["outcomes"][1]["turn_id"], "turn-1");
        assert!(log.snapshot_session("another-session")?.is_none());
        let contents = std::fs::read_to_string(&log.path)?;
        assert_eq!(contents.lines().count(), 3);
        let record: serde_json::Value =
            serde_json::from_str(contents.lines().next().expect("call record"))?;
        assert_eq!(record["session_id"], "session-1");
        assert_eq!(record["turn_id"], "turn-1");
        assert_eq!(record["target"], "judge");
        assert_eq!(record["model"], "judge-model");
        Ok(())
    }

    #[derive(Clone)]
    struct ShutdownTestState {
        started: Arc<Notify>,
        release: Arc<Notify>,
    }

    struct ShutdownTestServer {
        state: ShutdownTestState,
        shutdown: oneshot::Sender<()>,
        server: task::JoinHandle<ServerResult<()>>,
        request: task::JoinHandle<std::io::Result<Vec<u8>>>,
    }

    async fn blocked_request(State(state): State<ShutdownTestState>) -> &'static str {
        state.started.notify_one();
        state.release.notified().await;
        "done"
    }

    async fn raw_request(addr: SocketAddr) -> std::io::Result<Vec<u8>> {
        let mut stream = tokio::net::TcpStream::connect(addr).await?;
        stream
            .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
            .await?;
        let mut response = Vec::new();
        stream.read_to_end(&mut response).await?;
        Ok(response)
    }

    fn shutdown_test_server(shutdown_timeout: Duration) -> ShutdownTestServer {
        let state = ShutdownTestState {
            started: Arc::new(Notify::new()),
            release: Arc::new(Notify::new()),
        };
        let router = Router::new()
            .route("/", get(blocked_request))
            .with_state(state.clone());
        let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
            .expect("listener binds");
        let addr = listener.local_addr().expect("listener has an address");
        let (shutdown, shutdown_receiver) = oneshot::channel();
        let server = tokio::spawn(serve(listener, router, shutdown_timeout, async move {
            let _ = shutdown_receiver.await;
        }));
        let request = tokio::spawn(raw_request(addr));
        ShutdownTestServer {
            state,
            shutdown,
            server,
            request,
        }
    }

    // Active requests may finish within the grace period, while stuck requests are bounded.
    #[tokio::test]
    async fn shutdown_drains_until_configured_deadline() {
        let ShutdownTestServer {
            state,
            shutdown,
            mut server,
            request,
        } = shutdown_test_server(Duration::from_secs(1));
        state.started.notified().await;
        shutdown.send(()).expect("server receives shutdown");
        assert!(
            tokio::time::timeout(Duration::from_millis(25), &mut server)
                .await
                .is_err(),
            "server must wait for the active request"
        );
        state.release.notify_one();
        tokio::time::timeout(Duration::from_secs(1), server)
            .await
            .expect("server stops after request drains")
            .expect("server task completes")
            .expect("server exits cleanly");
        let response = request
            .await
            .expect("request task completes")
            .expect("request succeeds");
        assert!(response.windows(8).any(|part| part == b"200 OK\r\n"));
        assert!(response.ends_with(b"done"));

        let ShutdownTestServer {
            state,
            shutdown,
            server,
            request,
        } = shutdown_test_server(Duration::from_millis(25));
        state.started.notified().await;
        shutdown.send(()).expect("server receives shutdown");
        tokio::time::timeout(Duration::from_secs(1), server)
            .await
            .expect("shutdown deadline is enforced")
            .expect("server task completes")
            .expect("server exits cleanly");
        state.release.notify_one();
        request.abort();
    }

    #[derive(Clone)]
    struct CancelTestState {
        started: Arc<Notify>,
        cancelled: Arc<Notify>,
    }

    // Signals when the handler future is dropped, standing in for the in-flight
    // upstream call that a cancelled run releases.
    struct CancelSentinel(Arc<Notify>);

    impl Drop for CancelSentinel {
        fn drop(&mut self) {
            self.0.notify_one();
        }
    }

    async fn never_responds(State(state): State<CancelTestState>) -> &'static str {
        let _sentinel = CancelSentinel(state.cancelled.clone());
        state.started.notify_one();
        std::future::pending::<()>().await;
        "unreachable"
    }

    // A buffered request produces no response until the run finishes, so nothing is
    // written to the socket to reveal that the client left. The server must still drop
    // the handler future on disconnect, otherwise upstream work outlives its client.
    #[tokio::test]
    async fn client_disconnect_cancels_a_buffered_handler() {
        let state = CancelTestState {
            started: Arc::new(Notify::new()),
            cancelled: Arc::new(Notify::new()),
        };
        let router = Router::new()
            .route("/", post(never_responds))
            .with_state(state.clone());
        let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
            .expect("listener binds");
        let addr = listener.local_addr().expect("listener has an address");
        let (shutdown, shutdown_receiver) = oneshot::channel();
        let server = tokio::spawn(serve(
            listener,
            router,
            Duration::from_millis(25),
            async move {
                let _ = shutdown_receiver.await;
            },
        ));

        let mut stream = tokio::net::TcpStream::connect(addr)
            .await
            .expect("client connects");
        stream
            .write_all(b"POST / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}")
            .await
            .expect("client sends a complete request");
        state.started.notified().await;

        // The client goes away mid-run, before any response byte exists.
        drop(stream);

        tokio::time::timeout(Duration::from_secs(5), state.cancelled.notified())
            .await
            .expect("handler future is dropped when the client disconnects");

        let _ = shutdown.send(());
        let _ = tokio::time::timeout(Duration::from_secs(5), server).await;
    }

    // Collects the terminal request events emitted while it is the active subscriber.
    #[derive(Clone, Default)]
    struct CapturedEvents(Arc<Mutex<Vec<CapturedEvent>>>);

    struct CapturedEvent {
        level: Level,
        message: String,
    }

    impl tracing_subscriber::Layer<tracing_subscriber::Registry> for CapturedEvents {
        fn on_event(
            &self,
            event: &tracing::Event<'_>,
            _ctx: tracing_subscriber::layer::Context<'_, tracing_subscriber::Registry>,
        ) {
            if event.metadata().target() != "switchyard_server::request" {
                return;
            }
            let mut message = String::new();
            event.record(
                &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
                    message.push_str(&format!("{}={value:?} ", field.name()));
                },
            );
            self.0.lock().push(CapturedEvent {
                level: *event.metadata().level(),
                message,
            });
        }
    }

    fn request_log_context() -> RequestLogContext {
        RequestLogContext {
            started: Instant::now(),
            wire_format: WireFormat::OpenAiChat,
            requested_model: Some("switchyard/route".to_string()),
            streaming: false,
            session_id: Some("session".to_string()),
            correlation_id: None,
        }
    }

    fn captured_events(run: impl FnOnce()) -> Vec<CapturedEvent> {
        use tracing_subscriber::layer::SubscriberExt;

        let captured = CapturedEvents::default();
        let subscriber = tracing_subscriber::registry().with(captured.clone());
        crate::testing::with_subscriber(subscriber, run);
        std::mem::take(&mut *captured.0.lock())
    }

    // A dropped handler future is the only trace an abandoned buffered request leaves,
    // so the guard has to turn it into a terminal event rather than logging nothing.
    #[test]
    fn dropping_the_request_log_guard_reports_a_cancelled_request() {
        let events = captured_events(|| {
            drop(RequestLogGuard(Some(request_log_context())));
        });

        assert_eq!(events.len(), 1);
        assert_eq!(events[0].level, Level::WARN);
        assert!(
            events[0].message.contains("cancelled"),
            "{:?}",
            events[0].message
        );
    }

    // A request that reached a response is not a cancellation, even though the guard
    // is still dropped afterwards.
    #[test]
    fn an_answered_request_reports_only_its_response() {
        let events = captured_events(|| {
            let mut guard = RequestLogGuard(Some(request_log_context()));
            guard.emit(&StatusCode::OK.into_response());
        });

        assert_eq!(events.len(), 1);
        assert_eq!(events[0].level, Level::INFO);
        assert!(
            events[0].message.contains("handled"),
            "{:?}",
            events[0].message
        );
    }

    // An operator has to be able to tell "no session was sent" from "a blank one was
    // sent": an absent id omits the field, an empty one records an empty value.
    #[test]
    fn request_log_distinguishes_absent_ids_from_empty_ones() {
        let rows = [
            (None, None),
            (Some(String::new()), Some("\"\"")),
            (Some("abc".to_string()), Some("\"abc\"")),
        ];

        for (id, expected) in rows {
            let context = || {
                let mut context = request_log_context();
                context.session_id = id.clone();
                context.correlation_id = id.clone();
                context
            };

            let answered = captured_events(|| {
                let mut guard = RequestLogGuard(Some(context()));
                guard.emit(&StatusCode::OK.into_response());
            });
            let cancelled = captured_events(|| drop(RequestLogGuard(Some(context()))));

            for events in [answered, cancelled] {
                assert_eq!(events.len(), 1);
                for field in ["session_id", "correlation_id"] {
                    let rendered = events[0]
                        .message
                        .split_whitespace()
                        .find_map(|part| part.strip_prefix(&format!("{field}=")));
                    assert_eq!(rendered, expected, "{field} {id:?}: {}", events[0].message);
                }
            }
        }
    }

    // Terminal request severity follows HTTP status instead of error-path bookkeeping.
    #[test]
    fn request_log_level_follows_http_status() {
        assert_eq!(request_log_level(StatusCode::OK), Level::INFO);
        assert_eq!(request_log_level(StatusCode::BAD_REQUEST), Level::WARN);
        assert_eq!(
            request_log_level(StatusCode::INTERNAL_SERVER_ERROR),
            Level::ERROR
        );
    }

    // Canonical error text remains available without consuming the response body.
    #[test]
    fn error_response_carries_request_log_error() {
        let response = error_response(
            StatusCode::BAD_REQUEST,
            "invalid request",
            "invalid_request_error",
            "invalid_request_error",
        );

        assert_eq!(
            response
                .extensions()
                .get::<RequestLogError>()
                .map(|error| error.0.as_str()),
            Some("invalid request")
        );
    }

    // The provider's message can quote request content, so the request log
    // records only the error class while the client still sees the message.
    #[test]
    fn upstream_error_redacts_request_log_error() {
        const LEAKED: &str = "SECRET-quoted-request-content";
        let error = LlmClientError::UpstreamHttp {
            status: StatusCode::BAD_GATEWAY,
            body: format!(
                r#"{{"error":{{"message":"validation failed: {LEAKED}","code":"invalid_request_{LEAKED}"}}}}"#
            ),
        };
        for wire_format in [
            WireFormat::OpenAiChat,
            WireFormat::OpenAiResponses,
            WireFormat::AnthropicMessages,
        ] {
            let response = render_error_response(client_error(&error), wire_format);
            let events = captured_events(|| request_log_context().emit(&response));
            assert_eq!(events.len(), 1);
            assert!(!events[0].message.contains(LEAKED), "{}", events[0].message);
            assert!(
                events[0].message.contains("upstream_error"),
                "{}",
                events[0].message
            );
            let api_error = response
                .extensions()
                .get::<ApiError>()
                .expect("ApiError extension");
            assert!(api_error.message.contains(LEAKED), "{}", api_error.message);
            assert_eq!(api_error.code, format!("invalid_request_{LEAKED}"));
        }
    }

    // LiteLLM's cost header passes through to the client; auth headers do not.
    #[test]
    fn upstream_header_forwarding_covers_litellm_cost() {
        for name in [
            "baggage",
            "x-litellm-response-cost",
            "x-ratelimit-remaining",
            "x-upstream-retry",
        ] {
            let header: HeaderName = name.parse().expect("header name");
            assert!(should_forward_upstream_header(&header), "{name}");
        }
        let secret: HeaderName = "authorization".parse().expect("header name");
        assert!(!should_forward_upstream_header(&secret));
    }
}