Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

14.4 KB
Raw Blame
mod anthropic_messages;
mod facts;
mod openai_chat_completions;
mod openai_responses;
mod response;
mod sse;

use std::{
    collections::BTreeMap,
    time::{SystemTime, UNIX_EPOCH},
};

use axum::http::{HeaderMap, header};
use serde_json::{Map, Value, json, value::RawValue};
use tokio::time::Instant;

use crate::{
    correlation::RequestCorrelation,
    resolution::{ApiFormat, IngestionMode, MetadataValue, ResolvedRequestContext},
    telemetry,
};
use anthropic_messages::AnthropicMessagesCapture;
pub(crate) use facts::ProviderFacts;
pub(crate) use facts::{InferenceFacts, InputOmission, InputOmissionReason, RelayOutcome};
use openai_chat_completions::OpenAiChatCompletionsCapture;
use openai_responses::OpenAiResponsesCapture;

pub(crate) const MAX_INPUT_CAPTURE_BYTES: usize = 5 * 1024 * 1024;
pub(crate) const MAX_OUTPUT_CAPTURE_BYTES: usize = 1024 * 1024;
const MAX_ITEMS: usize = 256;
const MAX_FACT_STRING: usize = 512;
const MAX_PROJECTED_BYTES: usize = 64 * 1024;

enum ProtocolCapture {
    OpenAiResponses(OpenAiResponsesCapture),
    OpenAiChatCompletions(OpenAiChatCompletionsCapture),
    AnthropicMessages(AnthropicMessagesCapture),
}

impl ProtocolCapture {
    fn new(api_format: ApiFormat, headers: &HeaderMap, body: &[u8], mode: IngestionMode) -> Self {
        match api_format {
            ApiFormat::OpenAiResponses => {
                Self::OpenAiResponses(OpenAiResponsesCapture::new(headers, body, mode))
            }
            ApiFormat::OpenAiChatCompletions => {
                Self::OpenAiChatCompletions(OpenAiChatCompletionsCapture::new(headers, body, mode))
            }
            ApiFormat::AnthropicMessages => {
                Self::AnthropicMessages(AnthropicMessagesCapture::new(headers, body, mode))
            }
        }
    }

    fn record_response(&mut self, headers: &HeaderMap) {
        match self {
            Self::OpenAiResponses(capture) => capture.record_response(headers),
            Self::OpenAiChatCompletions(capture) => capture.record_response(headers),
            Self::AnthropicMessages(capture) => capture.record_response(headers),
        }
    }

    fn push_bytes(&mut self, bytes: &[u8]) -> bool {
        match self {
            Self::OpenAiResponses(capture) => capture.push_bytes(bytes),
            Self::OpenAiChatCompletions(capture) => capture.push_bytes(bytes),
            Self::AnthropicMessages(capture) => capture.push_bytes(bytes),
        }
    }

    fn end_body(&mut self) {
        match self {
            Self::OpenAiResponses(capture) => capture.end_body(),
            Self::OpenAiChatCompletions(capture) => capture.end_body(),
            Self::AnthropicMessages(capture) => capture.end_body(),
        }
    }

    fn into_facts(self) -> (&'static str, ProviderFacts) {
        match self {
            Self::OpenAiResponses(capture) => ("openai.responses", capture.into_facts()),
            Self::OpenAiChatCompletions(capture) => {
                ("openai.chat-completions", capture.into_facts())
            }
            Self::AnthropicMessages(capture) => ("anthropic.messages", capture.into_facts()),
        }
    }

    fn client_metadata(&self) -> Option<&Map<String, Value>> {
        match self {
            Self::OpenAiResponses(capture) => capture.client_metadata(),
            Self::OpenAiChatCompletions(_) | Self::AnthropicMessages(_) => None,
        }
    }
}

/// Owned before dispatch and moved into the response body. Drop also covers a
/// cancelled provider future before response headers have arrived.
pub(crate) struct ExecutionCapture {
    span: tracing::Span,
    protocol: Option<ProtocolCapture>,
    started: Instant,
    start_time_unix_ms: u128,
    first_byte_ms: Option<u128>,
    completion_start_ms: Option<u128>,
    http_status: Option<u16>,
    retry_after_ms: Option<u64>,
    request_body_bytes: usize,
    response_body_bytes: u64,
    metadata: Value,
    provider_model: Option<String>,
    delivery: Option<(telemetry::Telemetry, telemetry::DeliveryContext)>,
}

impl ExecutionCapture {
    pub(crate) fn unobserved() -> Self {
        Self {
            span: tracing::Span::current(),
            protocol: None,
            started: Instant::now(),
            start_time_unix_ms: SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .unwrap_or_default()
                .as_millis(),
            first_byte_ms: None,
            completion_start_ms: None,
            http_status: None,
            retry_after_ms: None,
            request_body_bytes: 0,
            response_body_bytes: 0,
            metadata: json!({}),
            provider_model: None,
            delivery: None,
        }
    }

    pub(crate) fn for_request(
        api_format: ApiFormat,
        context: &ResolvedRequestContext,
        headers: &HeaderMap,
        body: &[u8],
    ) -> Self {
        let started = Instant::now();
        let start_time_unix_ms = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap_or_default()
            .as_millis();
        let attribution = context.attribution();
        // Copy only attribution; never retain provider or ingestion credentials.
        // The entire resolver response, including key metadata, is bounded at 256 KiB.
        let key_metadata: Map<String, Value> = attribution
            .key_metadata()
            .iter()
            .map(|(key, value)| {
                let value = match value {
                    MetadataValue::String(value) => Value::String(value.clone()),
                    MetadataValue::Number(value) => Value::Number(value.clone()),
                    MetadataValue::Bool(value) => Value::Bool(*value),
                };
                (key.clone(), value)
            })
            .collect();
        let span = tracing::Span::current();
        // Parsing the whole request is CPU-bound and scales with the body.
        let protocol = tracing::info_span!(
            "request.capture",
            otel.kind = "internal",
            http.request.body.size = i64::try_from(body.len()).unwrap_or(i64::MAX)
        )
        .in_scope(|| ProtocolCapture::new(api_format, headers, body, context.ingestion_mode()));
        Self {
            span,
            protocol: Some(protocol),
            started,
            start_time_unix_ms,
            first_byte_ms: None,
            completion_start_ms: None,
            http_status: None,
            retry_after_ms: None,
            request_body_bytes: body.len(),
            response_body_bytes: 0,
            metadata: json!({
                "organization_id": attribution.organization_id().as_str(),
                "key_id": attribution.key_id(),
                "key_name": attribution.key_name().map(truncated_string),
                "key_owner_user_id": attribution.user_id(),
                "key_owner_name": attribution.user_name().and_then(bounded_string),
                "key_owner_email": attribution.user_email().and_then(bounded_string),
                "provider_connection_id": attribution.provider_connection_id(),
                "connection_name": context.connection().name().and_then(bounded_string),
                "provider": context.connection().provider().as_str(),
                "key_metadata": key_metadata,
                "ingestion_mode": context.ingestion_mode().as_str(),
            }),
            provider_model: None,
            delivery: None,
        }
    }

    pub(crate) fn deliver_to(
        &mut self,
        telemetry: telemetry::Telemetry,
        context: &ResolvedRequestContext,
        headers: &HeaderMap,
        correlation: &mut RequestCorrelation,
    ) {
        let client_metadata = self
            .protocol
            .as_ref()
            .and_then(ProtocolCapture::client_metadata);
        let Some(delivery) =
            telemetry::DeliveryContext::from_resolved(context, headers, client_metadata)
        else {
            return;
        };
        correlation.record_generation(delivery.generation_ids());
        self.metadata["request_id"] = json!(correlation.id());
        if let Some(client_id) = correlation.client_id() {
            self.metadata["client_request_id"] = json!(client_id);
        }
        self.delivery = Some((telemetry, delivery));
    }

    pub(crate) fn set_provider_model(&mut self, model: &str) {
        self.provider_model = bounded_string(model);
    }

    pub(crate) fn record_response(&mut self, status: u16, headers: &HeaderMap) {
        if let Some(protocol) = &mut self.protocol {
            self.http_status = Some(status);
            self.retry_after_ms = retry_after_ms(headers);
            protocol.record_response(headers);
        }
    }

    pub(crate) fn push_bytes(&mut self, bytes: &[u8]) {
        if let Some(protocol) = &mut self.protocol {
            self.response_body_bytes = self
                .response_body_bytes
                .saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX));
            if !bytes.is_empty() {
                self.first_byte_ms
                    .get_or_insert_with(|| self.started.elapsed().as_millis());
            }
            if protocol.push_bytes(bytes) {
                self.completion_start_ms
                    .get_or_insert_with(|| self.started.elapsed().as_millis());
            }
        }
    }

    pub(crate) fn end_body(&mut self) {
        if let Some(protocol) = &mut self.protocol {
            protocol.end_body();
        }
    }

    pub(crate) fn finish(&mut self, outcome: RelayOutcome) {
        let span = self.span.clone();
        let _entered = span.enter();
        crate::observability::execution_failed(outcome);
        let Some(protocol) = self.protocol.take() else {
            return;
        };
        let (api_format, inference) = protocol.into_facts();
        let facts = InferenceFacts {
            api_format,
            start_time_unix_ms: self.start_time_unix_ms,
            duration_ms: self.started.elapsed().as_millis(),
            first_byte_ms: self.first_byte_ms,
            completion_start_ms: self.completion_start_ms,
            http_status: self.http_status,
            retry_after_ms: self.retry_after_ms,
            upstream_model: self.provider_model.take(),
            request_body_bytes: self.request_body_bytes,
            response_body_bytes: self.response_body_bytes,
            metadata: std::mem::take(&mut self.metadata),
            outcome,
            inference,
        };
        telemetry::debug_record(&facts);
        crate::observability::execution_finished(&facts);
        if let Some((telemetry, context)) = self.delivery.take() {
            telemetry.record(context, facts);
        }
    }
}

impl Drop for ExecutionCapture {
    fn drop(&mut self) {
        self.finish(RelayOutcome::Cancelled);
    }
}

/// The request's top-level members, each kept as its original bytes. A parsed
/// `Value` tree can take about 100x the bytes of adversarial JSON such as
/// `[{"":0},...]`, so the members projected out of the input share one small
/// parse budget.
struct RawRequest {
    members: BTreeMap<String, Box<RawValue>>,
    parse_budget: usize,
}

impl RawRequest {
    /// Remove and parse a member that fits the remaining parse budget. Members
    /// that do not fit stay in the request and so in the recorded input.
    fn take(&mut self, key: &str) -> Option<Value> {
        self.take_where(key, |raw| serde_json::from_str(raw.get()).ok())
    }

    /// Remove a member that fits the remaining parse budget and that `parse` accepts.
    fn take_where<T>(
        &mut self,
        key: &str,
        parse: impl FnOnce(&RawValue) -> Option<T>,
    ) -> Option<T> {
        let raw = self.members.get(key)?;
        let remaining = self.parse_budget.checked_sub(raw.get().len())?;
        let value = parse(raw)?;
        self.parse_budget = remaining;
        self.members.remove(key);
        Some(value)
    }

    fn into_input(self) -> Option<Box<RawValue>> {
        serde_json::value::to_raw_value(&self.members).ok()
    }
}

fn parse_request(headers: &HeaderMap, body: &[u8]) -> Result<RawRequest, InputOmission> {
    let reason = if !identity_encoding(headers) {
        InputOmissionReason::ContentEncoding
    } else if body.len() > MAX_INPUT_CAPTURE_BYTES {
        InputOmissionReason::SizeLimit
    } else if let Ok(members) = serde_json::from_slice(body) {
        return Ok(RawRequest {
            members,
            parse_budget: MAX_PROJECTED_BYTES,
        });
    } else {
        InputOmissionReason::InvalidJson
    };
    Err(InputOmission {
        reason,
        body_bytes: body.len(),
    })
}

/// Only the delay-seconds form; an HTTP-date would make the value depend on clock skew.
fn retry_after_ms(headers: &HeaderMap) -> Option<u64> {
    headers
        .get(header::RETRY_AFTER)?
        .to_str()
        .ok()?
        .trim()
        .parse::<u64>()
        .ok()
        .map(|seconds| seconds.saturating_mul(1000))
}

fn bounded_string(value: &str) -> Option<String> {
    (value.len() <= MAX_FACT_STRING).then(|| value.to_owned())
}

/// Key names are the user ID fallback, so an over-long name is cut at a char
/// boundary instead of dropped. Web caps names at 500 UTF-16 code units, which
/// can exceed `MAX_FACT_STRING` bytes for non-Latin scripts.
fn truncated_string(value: &str) -> String {
    value[..value.floor_char_boundary(MAX_FACT_STRING)].to_owned()
}

fn identity_encoding(headers: &HeaderMap) -> bool {
    headers
        .get_all(header::CONTENT_ENCODING)
        .iter()
        .all(|v| v.to_str().is_ok_and(|v| v.eq_ignore_ascii_case("identity")))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn retry_after_accepts_only_delay_seconds() {
        for (value, expected) in [
            (Some("30"), Some(30_000)),
            (Some(" 0 "), Some(0)),
            (Some("Wed, 21 Oct 2026 07:28:00 GMT"), None),
            (Some("-1"), None),
            (Some("1.5"), None),
            (None, None),
        ] {
            let mut headers = HeaderMap::new();
            if let Some(value) = value {
                headers.insert(header::RETRY_AFTER, value.parse().unwrap());
            }
            assert_eq!(retry_after_ms(&headers), expected, "{value:?}");
        }
    }

    #[test]
    fn long_multibyte_key_names_are_truncated_not_dropped() {
        let name = "名".repeat(171);
        let truncated = truncated_string(&name);
        assert!(truncated.len() <= MAX_FACT_STRING);
        assert!(name.starts_with(&truncated));
        assert_eq!(truncated.chars().count(), MAX_FACT_STRING / 3);
        assert_eq!(truncated_string("checkout-agent"), "checkout-agent");
    }
}