Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

13.9 KB
Raw Blame
mod chat_completions;
pub(crate) mod model_list;

use std::{ops::Range, time::Duration};

use axum::{
    body::{Body, Bytes},
    http::{HeaderMap, HeaderName, HeaderValue, Response, StatusCode, header},
};
use reqwest::Client;
use reqwest_middleware::ClientWithMiddleware;
use reqwest_tracing::DisableOtelPropagation;
use tokio::time::Instant;

pub use crate::transport::ProviderError;
use crate::{
    capture::{ExecutionCapture, RelayOutcome},
    config::ExecutionLimits,
    correlation::{RequestCorrelation, UpstreamTarget},
    observability::{ErrorOrigin, RequestObservation, Wait, provider_status_reason},
    resolution::{ApiFormat, IngestionMode, Provider, ProviderCredential, ResolvedRequestContext},
    transport,
};

#[derive(Clone, Copy)]
pub(crate) enum Route {
    OpenAiResponses,
    OpenAiResponsesCompact,
    OpenAiChatCompletions,
    AnthropicMessages,
    AnthropicCountTokens,
}

impl Route {
    pub(crate) fn provider(self) -> Provider {
        match self {
            Self::OpenAiResponses | Self::OpenAiResponsesCompact | Self::OpenAiChatCompletions => {
                Provider::OpenAi
            }
            Self::AnthropicMessages | Self::AnthropicCountTokens => Provider::Anthropic,
        }
    }

    pub(crate) fn api_format(self) -> ApiFormat {
        match self {
            Self::OpenAiChatCompletions => ApiFormat::OpenAiChatCompletions,
            _ => match self.provider() {
                Provider::OpenAi => ApiFormat::OpenAiResponses,
                Provider::Anthropic => ApiFormat::AnthropicMessages,
            },
        }
    }

    fn path(self) -> &'static str {
        match self {
            Self::OpenAiResponses => "/responses",
            Self::OpenAiResponsesCompact => "/responses/compact",
            Self::OpenAiChatCompletions => "/chat/completions",
            Self::AnthropicMessages => "/messages",
            Self::AnthropicCountTokens => "/messages/count_tokens",
        }
    }

    pub(crate) fn captures_generation(self) -> bool {
        matches!(
            self,
            Self::OpenAiResponses
                | Self::OpenAiResponsesCompact
                | Self::OpenAiChatCompletions
                | Self::AnthropicMessages
        )
    }
}

#[derive(Default)]
pub(crate) struct ProviderLimits {
    pub execution: ExecutionLimits,
}

/// An admitted execution: its deadline and in-flight accounting. Memory is reserved
/// separately, before the body is read.
pub struct RequestPermit {
    _active: crate::observability::Active,
    deadline: Instant,
}

/// The validated caller body and the facts the relay needs about it.
pub(crate) struct RequestBody {
    pub bytes: Bytes,
    pub model_range: Range<usize>,
    /// The caller asked for a streamed response (`"stream": true`).
    pub streaming: bool,
    /// Capacity held for the body, released with the execution once the response ends.
    pub reservation: Option<BodyReservation>,
}

pub(crate) type BodyReservation = Box<dyn Send + Sync>;

pub struct ProviderTransport {
    client: ClientWithMiddleware,
    limits: ProviderLimits,
    telemetry: Option<crate::telemetry::Telemetry>,
    origin_override: Option<String>,
}

impl ProviderTransport {
    /// # Errors
    /// Returns [`ProviderError::Configuration`] when the HTTPS client cannot be initialized.
    pub fn new(execution: ExecutionLimits) -> Result<Self, ProviderError> {
        Self::with_limits(ProviderLimits { execution })
    }

    fn with_limits(limits: ProviderLimits) -> Result<Self, ProviderError> {
        let client = Client::builder()
            .redirect(reqwest::redirect::Policy::none())
            .retry(reqwest::retry::never())
            .no_proxy()
            .no_gzip()
            .no_brotli()
            .no_deflate()
            .no_zstd()
            // No client read timeout: it would also cut a non-streaming call while the
            // provider generates before sending headers. The relay bounds stalls instead.
            .connect_timeout(Duration::from_secs(5))
            .pool_max_idle_per_host(16)
            // reqwest's http2 feature is for Web-bound clients (resolver, telemetry); provider calls stay on HTTP/1.1.
            .http1_only()
            .build()
            .map_err(|_| ProviderError::Configuration)?;
        Ok(Self {
            client: crate::observability::instrument_client(client, "provider.headers"),
            limits,
            telemetry: None,
            origin_override: None,
        })
    }

    pub(crate) fn with_telemetry(mut self, telemetry: crate::telemetry::Telemetry) -> Self {
        self.telemetry = Some(telemetry);
        self
    }

    /// Start the execution deadline.
    pub fn admit(&self) -> RequestPermit {
        RequestPermit {
            _active: crate::observability::Active::new("execution"),
            deadline: Instant::now() + self.limits.execution.max_execution_time,
        }
    }

    /// Forward as a streamed `OpenAI` Responses call.
    ///
    /// # Errors
    /// See [`Self::forward_route`].
    #[cfg(test)]
    pub async fn forward(
        &self,
        permit: RequestPermit,
        context: ResolvedRequestContext,
        headers: &HeaderMap,
        body: Bytes,
    ) -> Result<Response<Body>, ProviderError> {
        let model_range = model_range(&body)?;
        self.forward_route(
            permit,
            context,
            headers,
            RequestBody {
                bytes: body,
                model_range,
                streaming: true,
                reservation: None,
            },
            Route::OpenAiResponses,
            &mut RequestCorrelation::from_headers(headers),
        )
        .await
    }

    pub(crate) async fn forward_route(
        &self,
        permit: RequestPermit,
        context: ResolvedRequestContext,
        headers: &HeaderMap,
        request: RequestBody,
        route: Route,
        correlation: &mut RequestCorrelation,
    ) -> Result<Response<Body>, ProviderError> {
        let RequestBody {
            bytes: body,
            model_range,
            streaming,
            reservation,
        } = request;
        let api_format = route.api_format();
        let (credential_name, mut credential) = provider_credential(&context)?;
        credential.set_sensitive(true);
        let provider_model = context.connection().provider_model();
        let mut capture = if route.captures_generation() {
            let mut capture = ExecutionCapture::for_request(api_format, &context, headers, &body);
            if let Some(telemetry) = &self.telemetry
                && context.ingestion_mode() != IngestionMode::None
            {
                capture.deliver_to(telemetry.clone(), &context, headers, correlation);
            }
            capture
        } else {
            ExecutionCapture::unobserved()
        };
        // Capture the caller's body shape, but keep upstream-only changes out of the input.
        let body = match route {
            Route::OpenAiChatCompletions => {
                match chat_completions::request_stream_usage(&body, provider_model) {
                    Some(rewritten) => {
                        drop(body);
                        rewritten
                    }
                    None => rewrite_model(body, model_range, provider_model)?,
                }
            }
            _ => rewrite_model(body, model_range, provider_model)?,
        };
        capture.set_provider_model(provider_model);
        correlation.record_upstream(UpstreamTarget::for_connection(
            context.connection(),
            Some(provider_model.to_owned()),
        ));
        let mut upstream = self
            .client
            .post(self.request_url(route))
            .with_extension(DisableOtelPropagation)
            .headers(transport::request_headers(headers, api_format))
            .header(header::ACCEPT_ENCODING, "identity")
            .header(credential_name, credential);
        upstream = upstream.body(body);
        let started = Instant::now();
        // A stream, or a call that does not generate, that goes quiet before its
        // headers is stalled. A non-streamed generation sends nothing until the whole
        // response is generated.
        let headers_deadline = if streaming || !route.captures_generation() {
            permit
                .deadline
                .min(Instant::now() + self.limits.execution.stall_timeout)
        } else {
            permit.deadline
        };
        let response = tokio::time::timeout_at(headers_deadline, upstream.send())
            .await
            .map_err(|_| ProviderError::Timeout)
            .and_then(|result| {
                result.map_err(|error| {
                    if error.is_timeout() {
                        ProviderError::Timeout
                    } else {
                        ProviderError::Transport
                    }
                })
            });
        let observation = correlation.observation();
        observation.record_wait(Wait::ProviderHeaders, started.elapsed());
        let response = match response {
            Ok(response) => response,
            Err(error) => {
                capture.finish(if matches!(error, ProviderError::Timeout) {
                    RelayOutcome::Timeout
                } else {
                    RelayOutcome::TransportError
                });
                return Err(error);
            }
        };
        if let Some(request_id) = transport::provider_request_id(response.headers(), api_format) {
            tracing::Span::current().record("provider_request_id", request_id);
        }
        let status = response.status();
        capture.record_response(status.as_u16(), response.headers());
        record_provider_status(observation, status);
        let mut downstream = Response::new(Body::empty());
        *downstream.status_mut() = status;
        *downstream.headers_mut() = transport::response_headers(response.headers(), api_format);
        *downstream.body_mut() = transport::relay(
            response,
            api_format.provider().as_str(),
            permit.deadline,
            self.limits.execution.stall_timeout,
            (permit, context, reservation),
            capture,
        );
        Ok(downstream)
    }

    /// Relay a gateway-built response body under the execution deadlines, holding
    /// `owner` until the client has read it or the relay ends.
    pub(crate) fn relay_buffered<T: Send + 'static>(&self, body: Vec<u8>, owner: T) -> Body {
        transport::relay_buffered(
            body,
            Instant::now() + self.limits.execution.max_execution_time,
            self.limits.execution.stall_timeout,
            owner,
        )
    }

    fn request_url(&self, route: Route) -> String {
        let base = self
            .origin_override
            .as_deref()
            .unwrap_or_else(|| route.provider().official_origin())
            .trim_end_matches('/');
        let mut url = String::with_capacity(base.len() + route.path().len());
        url.push_str(base);
        url.push_str(route.path());
        url
    }

    #[cfg(test)]
    pub(crate) fn for_test(base_url: String, limits: ProviderLimits) -> Self {
        let mut provider = Self::with_limits(limits).unwrap();
        provider.origin_override = Some(base_url);
        provider
    }
}

/// Provider error statuses are relayed unchanged, so the handler's rejection log
/// and error classification never see them.
fn record_provider_status(observation: &RequestObservation, status: StatusCode) {
    if !(status.is_client_error() || status.is_server_error()) {
        return;
    }
    let reason = provider_status_reason(status.as_u16());
    observation.record_error(ErrorOrigin::Provider, reason);
    if status.is_server_error() {
        tracing::warn!(
            reason,
            status = status.as_u16(),
            "provider returned an error status"
        );
    } else {
        tracing::info!(
            reason,
            status = status.as_u16(),
            "provider returned an error status"
        );
    }
}

#[expect(
    clippy::needless_pass_by_value,
    reason = "ownership releases the caller body when rewriting returns, before the upstream wait"
)]
fn rewrite_model(
    body: Bytes,
    model_range: Range<usize>,
    provider_model: &str,
) -> Result<Bytes, ProviderError> {
    if model_range.start > model_range.end || model_range.end > body.len() {
        return Err(ProviderError::Configuration);
    }
    let model = serde_json::to_vec(provider_model).map_err(|_| ProviderError::Configuration)?;
    let capacity = body.len() - model_range.len() + model.len();
    let mut rewritten = Vec::with_capacity(capacity);
    rewritten.extend_from_slice(&body[..model_range.start]);
    rewritten.extend_from_slice(&model);
    rewritten.extend_from_slice(&body[model_range.end..]);
    Ok(Bytes::from(rewritten))
}

#[cfg(test)]
pub(crate) fn model_range(body: &[u8]) -> Result<Range<usize>, ProviderError> {
    #[derive(serde::Deserialize)]
    struct ModelMember<'a> {
        #[serde(borrow)]
        model: &'a serde_json::value::RawValue,
    }

    let request = serde_json::from_slice::<ModelMember<'_>>(body)
        .map_err(|_| ProviderError::Configuration)?;
    let model = request.model.get();
    let start = model
        .as_ptr()
        .addr()
        .checked_sub(body.as_ptr().addr())
        .ok_or(ProviderError::Configuration)?;
    let end = start
        .checked_add(model.len())
        .filter(|end| *end <= body.len())
        .ok_or(ProviderError::Configuration)?;
    Ok(start..end)
}

fn provider_credential(
    context: &ResolvedRequestContext,
) -> Result<(HeaderName, HeaderValue), ProviderError> {
    let (name, value) = match context.connection().credential() {
        ProviderCredential::Bearer(token) => (header::AUTHORIZATION, format!("Bearer {token}")),
        ProviderCredential::XApiKey(key) => (HeaderName::from_static("x-api-key"), key.to_owned()),
    };
    let value = HeaderValue::from_str(&value).map_err(|_| ProviderError::Configuration)?;
    Ok((name, value))
}

#[cfg(test)]
mod tests;