Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

23.1 KB
Raw Blame
//! Caller tracing context belongs only to the recorded generation.
use std::collections::HashSet;

use axum::http::HeaderMap;
use opentelemetry::{Context, propagation::TextMapPropagator, trace::TraceContextExt};
use opentelemetry_http::HeaderExtractor;
use opentelemetry_sdk::{
    propagation::TraceContextPropagator,
    trace::{IdGenerator, RandomIdGenerator},
};
use serde::Serialize;
use serde_json::{Map, Value, value::RawValue};
use sha2::{Digest, Sha256};

const MAX_HEADER_BYTES: usize = 8 * 1024;
const MAX_FIELD_BYTES: usize = 1024;
const MAX_USER_AGENT_BYTES: usize = 256;
const MAX_ENTRIES: usize = 64;
const MAX_AGENT_HEADER_BYTES: usize = 8 * 1024;
const MAX_AGENT_METADATA_BYTES: usize = 4 * 1024;
/// Codex's body blob additionally carries its tool inventory, so it is bounded separately.
const MAX_CLIENT_METADATA_BYTES: usize = 64 * 1024;
const CODEX_TURN_METADATA_KEY: &str = "x-codex-turn-metadata";
/// Codex turn metadata retained as `agent.*`. Nested objects such as the tool inventory
/// and workspaces are not copied; the tool inventory already lives in the captured input.
const CODEX_STRING_FIELDS: [(&str, &str); 18] = [
    ("session_id", "agent.session_id"),
    ("thread_id", "agent.thread_id"),
    ("turn_id", "agent.turn_id"),
    ("agent_name", "agent.id"),
    ("installation_id", "agent.installation_id"),
    ("root_turn_id", "agent.root_turn_id"),
    ("parent_turn_id", "agent.parent_turn_id"),
    ("parent_thread_id", "agent.parent_thread_id"),
    ("forked_from_thread_id", "agent.forked_from_thread_id"),
    ("window_id", "agent.window_id"),
    ("context_window_id", "agent.context_window_id"),
    ("request_kind", "agent.request_kind"),
    ("subagent_kind", "agent.subagent_kind"),
    ("thread_source", "agent.thread_source"),
    ("turn_trigger", "agent.turn_trigger"),
    ("sandbox", "agent.sandbox"),
    ("sandbox_mode", "agent.sandbox_mode"),
    ("workspace_kind", "agent.workspace_kind"),
];
const CODEX_SCALAR_FIELDS: [(&str, &str); 4] = [
    ("window_number", "agent.window_number"),
    (
        "forked_from_ordinal_exclusive",
        "agent.forked_from_ordinal_exclusive",
    ),
    ("turn_started_at_unix_ms", "agent.turn_started_at_unix_ms"),
    ("auto_review_enabled", "agent.auto_review_enabled"),
];
const CODEX_COMPACTION_FIELDS: [&str; 5] =
    ["trigger", "reason", "implementation", "phase", "strategy"];
/// Flat `client_metadata` keys Codex projects from the same snapshot, used when the blob is absent.
const CODEX_FLAT_FIELDS: [(&str, &str); 8] = [
    ("session_id", "session_id"),
    ("thread_id", "thread_id"),
    ("turn_id", "turn_id"),
    ("root_turn_id", "root_turn_id"),
    ("parent_turn_id", "parent_turn_id"),
    ("x-codex-installation-id", "installation_id"),
    ("x-codex-window-id", "window_id"),
    ("x-codex-parent-thread-id", "parent_thread_id"),
];
const HEADERS: [&str; 9] = [
    "traceparent",
    "tracestate",
    "baggage",
    "langfuse-user-id",
    "langfuse-session-id",
    "langfuse-trace-name",
    "langfuse-environment",
    "langfuse-tags",
    "langfuse-metadata",
];
const SCALARS: [(&str, &str, &str); 4] = [
    ("langfuse-user-id", "langfuse_user_id", "user.id"),
    ("langfuse-session-id", "langfuse_session_id", "session.id"),
    (
        "langfuse-trace-name",
        "langfuse_trace_name",
        "langfuse.trace.name",
    ),
    (
        "langfuse-environment",
        "langfuse_environment",
        ENVIRONMENT_ATTRIBUTE,
    ),
];
const ENVIRONMENT_ATTRIBUTE: &str = "langfuse.environment";
/// Matches the SDKs' environment rule; ingestion stores `default` when none is set.
const MAX_ENVIRONMENT_BYTES: usize = 40;
const AGENT_HEADERS: [&str; 13] = [
    "user-agent",
    "x-claude-code-session-id",
    "x-claude-code-agent-id",
    "x-claude-code-parent-agent-id",
    "x-codex-turn-metadata",
    "x-opencode-project",
    "x-opencode-session",
    "x-opencode-request",
    "x-session-id",
    "x-session-affinity",
    "session_id",
    "x-client-request-id",
    "x-parent-session-id",
];

#[derive(Serialize)]
pub(super) struct GenerationContext {
    pub trace_id: String,
    pub observation_id: String,
    pub parent_span_id: Option<String>,
    pub trace_state: String,
    pub attributes: Map<String, Value>,
    pub metadata: Map<String, Value>,
    pub user_agent: Option<String>,
}

impl GenerationContext {
    #[cfg(test)]
    pub(super) fn from_headers(headers: &HeaderMap) -> Self {
        Self::from_request(headers, None)
    }

    /// `client_metadata` is the request body's agent metadata object, already
    /// removed from the captured input by [`agent_client_metadata`].
    pub(super) fn from_request(
        headers: &HeaderMap,
        client_metadata: Option<&Map<String, Value>>,
    ) -> Self {
        let ids = RandomIdGenerator::default();
        let mut result = Self {
            trace_id: ids.new_trace_id().to_string(),
            observation_id: ids.new_span_id().to_string(),
            parent_span_id: None,
            trace_state: String::new(),
            attributes: Map::new(),
            metadata: Map::new(),
            user_agent: client_user_agent(headers),
        };
        let bytes = HEADERS
            .iter()
            .flat_map(|name| headers.get_all(*name))
            .map(|value| value.as_bytes().len())
            .sum::<usize>();
        if bytes > MAX_HEADER_BYTES {
            return result;
        }
        let mut trace_headers = HeaderMap::new();
        if let Some(Ok(value)) = single_header(headers, "traceparent").map(str::parse) {
            trace_headers.insert("traceparent", value);
        }
        // W3C tracestate is a list and may span multiple HTTP header fields.
        let trace_state = headers
            .get_all("tracestate")
            .iter()
            .filter_map(|value| value.to_str().ok())
            .collect::<Vec<_>>()
            .join(",");
        if let Ok(value) = trace_state.parse() {
            trace_headers.insert("tracestate", value);
        }
        let context = TraceContextPropagator::new()
            .extract_with_context(&Context::new(), &HeaderExtractor(&trace_headers));
        let parent = context.span();
        let parent = parent.span_context();
        if parent.is_valid() && parent.is_remote() {
            result.trace_id = parent.trace_id().to_string();
            result.parent_span_id = Some(parent.span_id().to_string());
            result.trace_state = parent.trace_state().header();
        }
        result.apply_baggage(headers);
        for (header, _, attribute) in SCALARS {
            if let Some(value) = single_header(headers, header)
                .and_then(|v| decode(v, false))
                .filter(|v| valid_scalar(attribute, v))
            {
                result.attributes.insert(attribute.into(), value.into());
            }
        }
        result.apply_tags(
            list_entries(headers, "langfuse-tags")
                .filter_map(|tag| decode(tag.trim(), false))
                .collect(),
        );
        for entry in list_entries(headers, "langfuse-metadata") {
            if let Some((key, value)) = entry.split_once(':')
                && let (Some(key), Some(value)) =
                    (decode(key.trim(), false), decode(value.trim(), false))
            {
                result.metadata.insert(key, value.into());
            }
        }
        result.apply_agent(headers, client_metadata);
        result
    }

    fn apply_baggage(&mut self, headers: &HeaderMap) {
        // Do not use the generic baggage propagator: malformed input must never
        // be logged, and Python's exporter encodes spaces with quote_plus.
        for entry in list_entries(headers, "baggage") {
            let entry = entry.split(';').next().unwrap_or_default().trim();
            let Some((key, value)) = entry.split_once('=') else {
                continue;
            };
            let Some(key) = decode(key.trim(), true) else {
                continue;
            };
            let Some(value) = decode(value.trim(), true) else {
                continue;
            };
            if let Some((_, _, attribute)) = SCALARS.iter().find(|(_, name, _)| *name == key) {
                if valid_scalar(attribute, &value) {
                    self.attributes.insert((*attribute).into(), value.into());
                }
            } else if key == "langfuse_tags" {
                if let Some(tags) = parse_tags(&value) {
                    self.apply_tags(tags);
                }
            } else if let Some(key) = key.strip_prefix("langfuse_metadata_")
                && valid(key)
            {
                self.metadata.insert(key.into(), value.into());
            }
        }
    }

    fn apply_tags(&mut self, tags: Vec<String>) {
        let mut seen = HashSet::new();
        let tags: Vec<Value> = tags
            .into_iter()
            .take(MAX_ENTRIES)
            .filter(|tag| valid(tag) && seen.insert(tag.clone()))
            .map(Value::String)
            .collect();
        if !tags.is_empty() {
            self.attributes
                .insert("langfuse.trace.tags".into(), Value::Array(tags));
        }
    }

    fn apply_agent(&mut self, headers: &HeaderMap, client_metadata: Option<&Map<String, Value>>) {
        let Some(agent) = AgentContext::from_request(headers, client_metadata) else {
            return;
        };
        for (key, value) in agent.metadata {
            self.metadata.insert(key, value);
        }
        self.metadata.insert("agent.name".into(), agent.name.into());
        if let Some(session_id) = &agent.session_id
            && !self.attributes.contains_key("session.id")
        {
            self.attributes.insert(
                "session.id".into(),
                format!("{}:{session_id}", agent.name).into(),
            );
        }
        if !self.attributes.contains_key("langfuse.trace.name") {
            self.attributes
                .insert("langfuse.trace.name".into(), agent.name.into());
        }
        if self.parent_span_id.is_none()
            && let Some(turn_id) = &agent.turn_id
        {
            self.trace_id = agent_trace_id(agent.name, agent.session_id.as_deref(), turn_id);
        }
    }
}

struct AgentContext {
    name: &'static str,
    session_id: Option<String>,
    turn_id: Option<String>,
    metadata: Vec<(String, Value)>,
}

impl AgentContext {
    fn from_request(
        headers: &HeaderMap,
        client_metadata: Option<&Map<String, Value>>,
    ) -> Option<Self> {
        let bytes = AGENT_HEADERS
            .iter()
            .flat_map(|name| headers.get_all(*name))
            .map(|value| value.as_bytes().len())
            .sum::<usize>();
        if bytes > MAX_AGENT_HEADER_BYTES {
            return None;
        }
        Self::parse_claude_code(headers)
            .or_else(|| Self::parse_codex(headers, client_metadata))
            .or_else(|| Self::parse_opencode(headers))
            .or_else(|| Self::parse_pi(headers))
    }

    fn parse_claude_code(headers: &HeaderMap) -> Option<Self> {
        let session_id = agent_header(headers, "x-claude-code-session-id");
        let agent_id = agent_header(headers, "x-claude-code-agent-id");
        let parent_agent_id = agent_header(headers, "x-claude-code-parent-agent-id");
        if session_id.is_none() && agent_id.is_none() && parent_agent_id.is_none() {
            return None;
        }
        let mut metadata = Vec::new();
        push_metadata(&mut metadata, "agent.session_id", session_id.as_ref());
        push_metadata(&mut metadata, "agent.id", agent_id.as_ref());
        push_metadata(&mut metadata, "agent.parent_id", parent_agent_id.as_ref());
        Some(Self {
            name: "claude-code",
            session_id,
            turn_id: None,
            metadata,
        })
    }

    /// Codex sends its turn snapshot canonically as `client_metadata["x-codex-turn-metadata"]`
    /// in the request body; the same-named header and the flat `client_metadata` keys are
    /// compatibility projections of that snapshot.
    fn parse_codex(
        headers: &HeaderMap,
        client_metadata: Option<&Map<String, Value>>,
    ) -> Option<Self> {
        let header = single_header(headers, CODEX_TURN_METADATA_KEY);
        let client_metadata = client_metadata.filter(|metadata| is_codex_client_metadata(metadata));
        if header.is_none() && client_metadata.is_none() {
            return None;
        }
        let mut fields = client_metadata
            .and_then(|metadata| metadata.get(CODEX_TURN_METADATA_KEY)?.as_str())
            .and_then(|blob| parse_turn_metadata(blob, MAX_CLIENT_METADATA_BYTES))
            .or_else(|| header.and_then(|blob| parse_turn_metadata(blob, MAX_AGENT_METADATA_BYTES)))
            .unwrap_or_default();
        for (source, target) in CODEX_FLAT_FIELDS {
            if let Some(value) = client_metadata.and_then(|metadata| metadata.get(source))
                && value.is_string()
                && !fields.contains_key(target)
            {
                fields.insert(target.to_owned(), value.clone());
            }
        }
        let session_id = agent_json_field(&fields, "session_id");
        let thread_id = agent_json_field(&fields, "thread_id");
        let turn_id = agent_json_field(&fields, "turn_id");
        if session_id.is_none() && thread_id.is_none() && turn_id.is_none() {
            return None;
        }
        let mut metadata = Vec::new();
        for (source, target) in CODEX_STRING_FIELDS {
            if let Some(value) = agent_json_field(&fields, source) {
                metadata.push((target.to_owned(), Value::String(value)));
            }
        }
        for (source, target) in CODEX_SCALAR_FIELDS {
            if let Some(value) = fields
                .get(source)
                .filter(|value| value.is_number() || value.is_boolean())
            {
                metadata.push((target.to_owned(), value.clone()));
            }
        }
        if let Some(compaction) = fields.get("compaction").and_then(Value::as_object) {
            for key in CODEX_COMPACTION_FIELDS {
                if let Some(value) = agent_json_field(compaction, key) {
                    metadata.push((format!("agent.compaction.{key}"), Value::String(value)));
                }
            }
        }
        Some(Self {
            name: "codex",
            session_id: thread_id,
            turn_id,
            metadata,
        })
    }

    fn parse_opencode(headers: &HeaderMap) -> Option<Self> {
        let project_id = agent_header(headers, "x-opencode-project");
        let direct_session_id = agent_header(headers, "x-opencode-session");
        let turn_id = agent_header(headers, "x-opencode-request");
        let parent_session_id = agent_header(headers, "x-parent-session-id");
        let identified = project_id.is_some()
            || direct_session_id.is_some()
            || turn_id.is_some()
            || user_agent(headers).is_some_and(|value| value.starts_with("opencode/"));
        if !identified {
            return None;
        }
        let session_id = direct_session_id
            .or_else(|| agent_header(headers, "x-session-id"))
            .or_else(|| agent_header(headers, "x-session-affinity"));
        let mut metadata = Vec::new();
        push_metadata(&mut metadata, "agent.project_id", project_id.as_ref());
        push_metadata(&mut metadata, "agent.session_id", session_id.as_ref());
        push_metadata(&mut metadata, "agent.turn_id", turn_id.as_ref());
        push_metadata(
            &mut metadata,
            "agent.parent_session_id",
            parent_session_id.as_ref(),
        );
        Some(Self {
            name: "opencode",
            session_id,
            turn_id,
            metadata,
        })
    }

    fn parse_pi(headers: &HeaderMap) -> Option<Self> {
        let identified = user_agent(headers)
            .is_some_and(|value| value.starts_with("pi/") || value.starts_with("pi ("));
        if !identified {
            return None;
        }
        let session_id = agent_header(headers, "x-session-id")
            .or_else(|| agent_header(headers, "session_id"))
            .or_else(|| agent_header(headers, "x-session-affinity"))
            .or_else(|| agent_header(headers, "x-client-request-id"));
        let mut metadata = Vec::new();
        push_metadata(&mut metadata, "agent.session_id", session_id.as_ref());
        Some(Self {
            name: "pi",
            session_id,
            turn_id: None,
            metadata,
        })
    }
}

/// A coding agent's `client_metadata` object, parsed so its identifiers are recorded
/// as `agent.*` metadata rather than as prompt input. Unrecognized or oversized
/// objects yield `None` and stay in the captured request untouched.
pub(crate) fn agent_client_metadata(raw: &RawValue) -> Option<Map<String, Value>> {
    if raw.get().len() > MAX_CLIENT_METADATA_BYTES {
        return None;
    }
    serde_json::from_str(raw.get())
        .ok()
        .filter(is_codex_client_metadata)
}

fn is_codex_client_metadata(metadata: &Map<String, Value>) -> bool {
    metadata.keys().any(|key| key.starts_with("x-codex-"))
}

fn parse_turn_metadata(blob: &str, limit: usize) -> Option<Map<String, Value>> {
    if blob.len() > limit {
        return None;
    }
    match serde_json::from_str::<Value>(blob).ok()? {
        Value::Object(fields) => Some(fields),
        _ => None,
    }
}

fn push_metadata(metadata: &mut Vec<(String, Value)>, key: &str, value: Option<&String>) {
    if let Some(value) = value {
        metadata.push((key.to_owned(), Value::String(value.clone())));
    }
}

fn agent_json_field(value: &Map<String, Value>, key: &str) -> Option<String> {
    clean_agent_field(value.get(key)?.as_str()?)
}

fn agent_header(headers: &HeaderMap, name: &str) -> Option<String> {
    clean_agent_field(single_header(headers, name)?)
}

fn clean_agent_field(value: &str) -> Option<String> {
    let value = value.trim();
    valid(value).then(|| value.to_owned())
}

fn user_agent(headers: &HeaderMap) -> Option<String> {
    agent_header(headers, "user-agent").map(|value| value.to_ascii_lowercase())
}

fn client_user_agent(headers: &HeaderMap) -> Option<String> {
    let value = single_header(headers, "user-agent")?.trim();
    if value.is_empty() || value.chars().any(char::is_control) {
        return None;
    }
    let mut end = value.len().min(MAX_USER_AGENT_BYTES);
    while !value.is_char_boundary(end) {
        end -= 1;
    }
    Some(value[..end].to_owned())
}

fn agent_trace_id(name: &str, session_id: Option<&str>, turn_id: &str) -> String {
    let mut hasher = Sha256::new();
    hasher.update(b"langfuse-agent-turn-v1\0");
    hasher.update(name.as_bytes());
    hasher.update(b"\0");
    if let Some(session_id) = session_id {
        hasher.update(session_id.as_bytes());
    }
    hasher.update(b"\0");
    hasher.update(turn_id.as_bytes());
    let digest = hasher.finalize();
    let mut trace_id = [0; 16];
    trace_id.copy_from_slice(&digest[..16]);
    if trace_id.iter().all(|byte| *byte == 0) {
        trace_id[15] = 1;
    }
    opentelemetry::trace::TraceId::from_bytes(trace_id).to_string()
}

fn list_entries<'a>(headers: &'a HeaderMap, name: &str) -> impl Iterator<Item = &'a str> {
    headers
        .get_all(name)
        .into_iter()
        .filter_map(|value| value.to_str().ok())
        .flat_map(|value| value.split(','))
        .take(MAX_ENTRIES)
}

fn single_header<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> {
    let mut values = headers.get_all(name).iter();
    let value = values.next()?;
    if values.next().is_some() {
        return None;
    }
    value.to_str().ok()
}

fn valid(value: &str) -> bool {
    !value.trim().is_empty()
        && value.len() <= MAX_FIELD_BYTES
        && !value.chars().any(char::is_control)
}

fn valid_scalar(attribute: &str, value: &str) -> bool {
    attribute != ENVIRONMENT_ATTRIBUTE || valid_environment(value)
}

/// Lowercase alphanumerics, `-` and `_`, at most 40 bytes; the `langfuse`
/// prefix is reserved for Langfuse-internal traces.
fn valid_environment(value: &str) -> bool {
    !value.is_empty()
        && value.len() <= MAX_ENVIRONMENT_BYTES
        && !value.starts_with("langfuse")
        && value
            .bytes()
            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_".contains(&byte))
}

fn decode(value: &str, plus_as_space: bool) -> Option<String> {
    let mut decoded = Vec::with_capacity(value.len().min(MAX_FIELD_BYTES));
    let mut bytes = value.bytes();
    while let Some(byte) = bytes.next() {
        let byte = match byte {
            b'%' => {
                let high = char::from(bytes.next()?).to_digit(16)?;
                let low = char::from(bytes.next()?).to_digit(16)?;
                u8::try_from(high * 16 + low).ok()?
            }
            b'+' if plus_as_space => b' ',
            _ => byte,
        };
        decoded.push(byte);
        if decoded.len() > MAX_FIELD_BYTES {
            return None;
        }
    }
    let value = String::from_utf8(decoded).ok()?;
    valid(&value).then_some(value)
}

/// Python's `OTel` exporter calls `str(list)`, producing quoted Python strings.
/// Accept only a list of string literals; never evaluate expressions.
fn parse_tags(value: &str) -> Option<Vec<String>> {
    if let Ok(tags) = serde_json::from_str::<Vec<String>>(value) {
        return (tags.len() <= MAX_ENTRIES).then_some(tags);
    }
    let mut chars = value.trim().chars().peekable();
    if chars.next()? != '[' {
        return None;
    }
    let mut tags = Vec::new();
    loop {
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        if chars.peek() == Some(&']') {
            chars.next();
            return chars.next().is_none().then_some(tags);
        }
        if tags.len() == MAX_ENTRIES {
            return None;
        }
        let quote = chars.next()?;
        if quote != '\'' && quote != '"' {
            return None;
        }
        let mut tag = String::new();
        loop {
            let character = chars.next()?;
            if character == quote {
                break;
            }
            if character != '\\' {
                tag.push(character);
                continue;
            }
            let escape = chars.next()?;
            tag.push(match escape {
                '\\' | '\'' | '"' => escape,
                'a' => '\u{7}',
                'b' => '\u{8}',
                'f' => '\u{c}',
                'n' => '\n',
                'r' => '\r',
                't' => '\t',
                'v' => '\u{b}',
                'x' | 'u' | 'U' => {
                    let digits = match escape {
                        'x' => 2,
                        'u' => 4,
                        _ => 8,
                    };
                    let mut code = 0;
                    for _ in 0..digits {
                        code = code * 16 + chars.next()?.to_digit(16)?;
                    }
                    char::from_u32(code)?
                }
                _ => return None,
            });
        }
        tags.push(tag);
        while chars.peek().is_some_and(|c| c.is_whitespace()) {
            chars.next();
        }
        match chars.peek()? {
            ',' => {
                chars.next();
            }
            ']' => {}
            _ => return None,
        }
    }
}

#[cfg(test)]
mod tests;