🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
mod anthropic_messages;
mod facts;
mod openai_chat_completions;
mod openai_responses;
mod response;
mod sse;
use std::{
collections::BTreeMap,
time::{SystemTime, UNIX_EPOCH},
};
use axum::http::{HeaderMap, header};
use serde_json::{Map, Value, json, value::RawValue};
use tokio::time::Instant;
use crate::{
correlation::RequestCorrelation,
resolution::{ApiFormat, IngestionMode, MetadataValue, ResolvedRequestContext},
telemetry,
};
use anthropic_messages::AnthropicMessagesCapture;
pub(crate) use facts::ProviderFacts;
pub(crate) use facts::{InferenceFacts, InputOmission, InputOmissionReason, RelayOutcome};
use openai_chat_completions::OpenAiChatCompletionsCapture;
use openai_responses::OpenAiResponsesCapture;
pub(crate) const MAX_INPUT_CAPTURE_BYTES: usize = 5 * 1024 * 1024;
pub(crate) const MAX_OUTPUT_CAPTURE_BYTES: usize = 1024 * 1024;
const MAX_ITEMS: usize = 256;
const MAX_FACT_STRING: usize = 512;
const MAX_PROJECTED_BYTES: usize = 64 * 1024;
enum ProtocolCapture {
OpenAiResponses(OpenAiResponsesCapture),
OpenAiChatCompletions(OpenAiChatCompletionsCapture),
AnthropicMessages(AnthropicMessagesCapture),
}
impl ProtocolCapture {
fn new(api_format: ApiFormat, headers: &HeaderMap, body: &[u8], mode: IngestionMode) -> Self {
match api_format {
ApiFormat::OpenAiResponses => {
Self::OpenAiResponses(OpenAiResponsesCapture::new(headers, body, mode))
}
ApiFormat::OpenAiChatCompletions => {
Self::OpenAiChatCompletions(OpenAiChatCompletionsCapture::new(headers, body, mode))
}
ApiFormat::AnthropicMessages => {
Self::AnthropicMessages(AnthropicMessagesCapture::new(headers, body, mode))
}
}
}
fn record_response(&mut self, headers: &HeaderMap) {
match self {
Self::OpenAiResponses(capture) => capture.record_response(headers),
Self::OpenAiChatCompletions(capture) => capture.record_response(headers),
Self::AnthropicMessages(capture) => capture.record_response(headers),
}
}
fn push_bytes(&mut self, bytes: &[u8]) -> bool {
match self {
Self::OpenAiResponses(capture) => capture.push_bytes(bytes),
Self::OpenAiChatCompletions(capture) => capture.push_bytes(bytes),
Self::AnthropicMessages(capture) => capture.push_bytes(bytes),
}
}
fn end_body(&mut self) {
match self {
Self::OpenAiResponses(capture) => capture.end_body(),
Self::OpenAiChatCompletions(capture) => capture.end_body(),
Self::AnthropicMessages(capture) => capture.end_body(),
}
}
fn into_facts(self) -> (&'static str, ProviderFacts) {
match self {
Self::OpenAiResponses(capture) => ("openai.responses", capture.into_facts()),
Self::OpenAiChatCompletions(capture) => {
("openai.chat-completions", capture.into_facts())
}
Self::AnthropicMessages(capture) => ("anthropic.messages", capture.into_facts()),
}
}
fn client_metadata(&self) -> Option<&Map<String, Value>> {
match self {
Self::OpenAiResponses(capture) => capture.client_metadata(),
Self::OpenAiChatCompletions(_) | Self::AnthropicMessages(_) => None,
}
}
}
/// Owned before dispatch and moved into the response body. Drop also covers a
/// cancelled provider future before response headers have arrived.
pub(crate) struct ExecutionCapture {
span: tracing::Span,
protocol: Option<ProtocolCapture>,
started: Instant,
start_time_unix_ms: u128,
first_byte_ms: Option<u128>,
completion_start_ms: Option<u128>,
http_status: Option<u16>,
retry_after_ms: Option<u64>,
request_body_bytes: usize,
response_body_bytes: u64,
metadata: Value,
provider_model: Option<String>,
delivery: Option<(telemetry::Telemetry, telemetry::DeliveryContext)>,
}
impl ExecutionCapture {
pub(crate) fn unobserved() -> Self {
Self {
span: tracing::Span::current(),
protocol: None,
started: Instant::now(),
start_time_unix_ms: SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
first_byte_ms: None,
completion_start_ms: None,
http_status: None,
retry_after_ms: None,
request_body_bytes: 0,
response_body_bytes: 0,
metadata: json!({}),
provider_model: None,
delivery: None,
}
}
pub(crate) fn for_request(
api_format: ApiFormat,
context: &ResolvedRequestContext,
headers: &HeaderMap,
body: &[u8],
) -> Self {
let started = Instant::now();
let start_time_unix_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
let attribution = context.attribution();
// Copy only attribution; never retain provider or ingestion credentials.
// The entire resolver response, including key metadata, is bounded at 256 KiB.
let key_metadata: Map<String, Value> = attribution
.key_metadata()
.iter()
.map(|(key, value)| {
let value = match value {
MetadataValue::String(value) => Value::String(value.clone()),
MetadataValue::Number(value) => Value::Number(value.clone()),
MetadataValue::Bool(value) => Value::Bool(*value),
};
(key.clone(), value)
})
.collect();
let span = tracing::Span::current();
// Parsing the whole request is CPU-bound and scales with the body.
let protocol = tracing::info_span!(
"request.capture",
otel.kind = "internal",
http.request.body.size = i64::try_from(body.len()).unwrap_or(i64::MAX)
)
.in_scope(|| ProtocolCapture::new(api_format, headers, body, context.ingestion_mode()));
Self {
span,
protocol: Some(protocol),
started,
start_time_unix_ms,
first_byte_ms: None,
completion_start_ms: None,
http_status: None,
retry_after_ms: None,
request_body_bytes: body.len(),
response_body_bytes: 0,
metadata: json!({
"organization_id": attribution.organization_id().as_str(),
"key_id": attribution.key_id(),
"key_name": attribution.key_name().map(truncated_string),
"key_owner_user_id": attribution.user_id(),
"key_owner_name": attribution.user_name().and_then(bounded_string),
"key_owner_email": attribution.user_email().and_then(bounded_string),
"provider_connection_id": attribution.provider_connection_id(),
"connection_name": context.connection().name().and_then(bounded_string),
"provider": context.connection().provider().as_str(),
"key_metadata": key_metadata,
"ingestion_mode": context.ingestion_mode().as_str(),
}),
provider_model: None,
delivery: None,
}
}
pub(crate) fn deliver_to(
&mut self,
telemetry: telemetry::Telemetry,
context: &ResolvedRequestContext,
headers: &HeaderMap,
correlation: &mut RequestCorrelation,
) {
let client_metadata = self
.protocol
.as_ref()
.and_then(ProtocolCapture::client_metadata);
let Some(delivery) =
telemetry::DeliveryContext::from_resolved(context, headers, client_metadata)
else {
return;
};
correlation.record_generation(delivery.generation_ids());
self.metadata["request_id"] = json!(correlation.id());
if let Some(client_id) = correlation.client_id() {
self.metadata["client_request_id"] = json!(client_id);
}
self.delivery = Some((telemetry, delivery));
}
pub(crate) fn set_provider_model(&mut self, model: &str) {
self.provider_model = bounded_string(model);
}
pub(crate) fn record_response(&mut self, status: u16, headers: &HeaderMap) {
if let Some(protocol) = &mut self.protocol {
self.http_status = Some(status);
self.retry_after_ms = retry_after_ms(headers);
protocol.record_response(headers);
}
}
pub(crate) fn push_bytes(&mut self, bytes: &[u8]) {
if let Some(protocol) = &mut self.protocol {
self.response_body_bytes = self
.response_body_bytes
.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX));
if !bytes.is_empty() {
self.first_byte_ms
.get_or_insert_with(|| self.started.elapsed().as_millis());
}
if protocol.push_bytes(bytes) {
self.completion_start_ms
.get_or_insert_with(|| self.started.elapsed().as_millis());
}
}
}
pub(crate) fn end_body(&mut self) {
if let Some(protocol) = &mut self.protocol {
protocol.end_body();
}
}
pub(crate) fn finish(&mut self, outcome: RelayOutcome) {
let span = self.span.clone();
let _entered = span.enter();
crate::observability::execution_failed(outcome);
let Some(protocol) = self.protocol.take() else {
return;
};
let (api_format, inference) = protocol.into_facts();
let facts = InferenceFacts {
api_format,
start_time_unix_ms: self.start_time_unix_ms,
duration_ms: self.started.elapsed().as_millis(),
first_byte_ms: self.first_byte_ms,
completion_start_ms: self.completion_start_ms,
http_status: self.http_status,
retry_after_ms: self.retry_after_ms,
upstream_model: self.provider_model.take(),
request_body_bytes: self.request_body_bytes,
response_body_bytes: self.response_body_bytes,
metadata: std::mem::take(&mut self.metadata),
outcome,
inference,
};
telemetry::debug_record(&facts);
crate::observability::execution_finished(&facts);
if let Some((telemetry, context)) = self.delivery.take() {
telemetry.record(context, facts);
}
}
}
impl Drop for ExecutionCapture {
fn drop(&mut self) {
self.finish(RelayOutcome::Cancelled);
}
}
/// The request's top-level members, each kept as its original bytes. A parsed
/// `Value` tree can take about 100x the bytes of adversarial JSON such as
/// `[{"":0},...]`, so the members projected out of the input share one small
/// parse budget.
struct RawRequest {
members: BTreeMap<String, Box<RawValue>>,
parse_budget: usize,
}
impl RawRequest {
/// Remove and parse a member that fits the remaining parse budget. Members
/// that do not fit stay in the request and so in the recorded input.
fn take(&mut self, key: &str) -> Option<Value> {
self.take_where(key, |raw| serde_json::from_str(raw.get()).ok())
}
/// Remove a member that fits the remaining parse budget and that `parse` accepts.
fn take_where<T>(
&mut self,
key: &str,
parse: impl FnOnce(&RawValue) -> Option<T>,
) -> Option<T> {
let raw = self.members.get(key)?;
let remaining = self.parse_budget.checked_sub(raw.get().len())?;
let value = parse(raw)?;
self.parse_budget = remaining;
self.members.remove(key);
Some(value)
}
fn into_input(self) -> Option<Box<RawValue>> {
serde_json::value::to_raw_value(&self.members).ok()
}
}
fn parse_request(headers: &HeaderMap, body: &[u8]) -> Result<RawRequest, InputOmission> {
let reason = if !identity_encoding(headers) {
InputOmissionReason::ContentEncoding
} else if body.len() > MAX_INPUT_CAPTURE_BYTES {
InputOmissionReason::SizeLimit
} else if let Ok(members) = serde_json::from_slice(body) {
return Ok(RawRequest {
members,
parse_budget: MAX_PROJECTED_BYTES,
});
} else {
InputOmissionReason::InvalidJson
};
Err(InputOmission {
reason,
body_bytes: body.len(),
})
}
/// Only the delay-seconds form; an HTTP-date would make the value depend on clock skew.
fn retry_after_ms(headers: &HeaderMap) -> Option<u64> {
headers
.get(header::RETRY_AFTER)?
.to_str()
.ok()?
.trim()
.parse::<u64>()
.ok()
.map(|seconds| seconds.saturating_mul(1000))
}
fn bounded_string(value: &str) -> Option<String> {
(value.len() <= MAX_FACT_STRING).then(|| value.to_owned())
}
/// Key names are the user ID fallback, so an over-long name is cut at a char
/// boundary instead of dropped. Web caps names at 500 UTF-16 code units, which
/// can exceed `MAX_FACT_STRING` bytes for non-Latin scripts.
fn truncated_string(value: &str) -> String {
value[..value.floor_char_boundary(MAX_FACT_STRING)].to_owned()
}
fn identity_encoding(headers: &HeaderMap) -> bool {
headers
.get_all(header::CONTENT_ENCODING)
.iter()
.all(|v| v.to_str().is_ok_and(|v| v.eq_ignore_ascii_case("identity")))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_after_accepts_only_delay_seconds() {
for (value, expected) in [
(Some("30"), Some(30_000)),
(Some(" 0 "), Some(0)),
(Some("Wed, 21 Oct 2026 07:28:00 GMT"), None),
(Some("-1"), None),
(Some("1.5"), None),
(None, None),
] {
let mut headers = HeaderMap::new();
if let Some(value) = value {
headers.insert(header::RETRY_AFTER, value.parse().unwrap());
}
assert_eq!(retry_after_ms(&headers), expected, "{value:?}");
}
}
#[test]
fn long_multibyte_key_names_are_truncated_not_dropped() {
let name = "名".repeat(171);
let truncated = truncated_string(&name);
assert!(truncated.len() <= MAX_FACT_STRING);
assert!(name.starts_with(&truncated));
assert_eq!(truncated.chars().count(), MAX_FACT_STRING / 3);
assert_eq!(truncated_string("checkout-agent"), "checkout-agent");
}
}