:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
mod attribution;
mod error_status;
mod fan_out;
mod filtering;
mod identity;
mod ingestion;
mod provenance;
mod providers;
use axum::body::Body;
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Json, Response};
use axum_client_ip::InsecureClientIp;
use chrono::Utc;
use metrics::{counter, histogram};
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use serde_json::json;
use tracing::{debug, instrument, warn, Span};
use crate::api::{CaptureError, CaptureResponse, CaptureResponseCode};
use crate::events::ai_byte_limit::drop_ai_byte_limited;
use crate::events::overflow_stamping::stamp_overflow_reason;
use crate::extractors::extract_body_with_timeout;
use crate::ingestion_warnings::otel::{
emit_no_ai_spans_warning, emit_otel_parse_warning, emit_span_cap_warning,
emit_span_too_big_warning, SpanCapStage,
};
use crate::prometheus::{report_dropped_events, report_internal_error_metrics};
use crate::router::State as AppState;
use crate::token::validate_token;
use crate::v0_request::exceeds_max_ai_event_bytes;
use self::attribution::otel_request_context;
pub const OTEL_BODY_SIZE: usize = 4 * 1024 * 1024; // 4MB
/// Route this handler serves. Stamped onto warnings so a reader of the v2 table
/// can tell which endpoint produced them.
const OTEL_PATH: &str = "/i/v0/ai/otel";
/// Maximum AI spans accepted after filtering. SDKs receive a 400 (non-retryable)
/// if this is exceeded, so callers must batch sensibly.
const MAX_SPANS_PER_REQUEST: usize = 100;
/// Maximum raw spans accepted before filtering. Set well above MAX_SPANS_PER_REQUEST
/// to accommodate mixed-content batches (e.g. Next.js sending HTTP + AI spans together)
/// while still bounding the cost of attribute scanning.
const MAX_RAW_SPANS_PER_REQUEST: usize = 1000;
fn count_spans(request: &ExportTraceServiceRequest) -> usize {
request
.resource_spans
.iter()
.flat_map(|rs| &rs.scope_spans)
.map(|ss| ss.spans.len())
.sum()
}
/// Return an HTTP 400 with a JSON error message.
///
/// Per the OTLP spec (https://opentelemetry.io/docs/specs/otlp/#failures-1), only
/// 429/502/503/504 are retryable — all other 4xx cause the SDK to permanently drop
/// the data. We use 400 (not 429) for quota/restriction rejections because we don't
/// want SDKs to retry data that will always be rejected.
fn non_retryable_rejection(message: &str) -> Response {
(StatusCode::BAD_REQUEST, Json(json!({ "error": message }))).into_response()
}
#[instrument(skip(state, body), fields(span_count, body_size))]
pub async fn otel_handler(
State(state): State<AppState>,
ip: Option<InsecureClientIp>,
headers: HeaderMap,
body: Body,
) -> Result<Json<serde_json::Value>, Response> {
let body = extract_body_with_timeout(
body,
OTEL_BODY_SIZE,
state.body_chunk_read_timeout,
state.body_read_chunk_size_kb,
OTEL_PATH,
)
.await
.map_err(|e| {
report_internal_error_metrics(e.to_metric_tag(), "otel_body_read");
e.into_response()
})?;
if body.is_empty() {
let err = CaptureError::EmptyPayload;
report_internal_error_metrics(err.to_metric_tag(), "otel_validation");
return Err(err.into_response());
}
let body_len = body.len();
Span::current().record("body_size", body_len);
histogram!("capture_ai_otel_body_size_bytes").record(body_len as f64);
let content_type = headers
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown");
let format = if content_type.starts_with("application/x-protobuf") {
"protobuf"
} else if content_type.starts_with("application/json") {
"json"
} else {
"unknown"
};
let normalized_content_type = match format {
"protobuf" => "application/x-protobuf",
"json" => "application/json",
_ => "unknown",
};
let content_encoding = headers
.get("content-encoding")
.and_then(|value| value.to_str().ok())
.map(str::trim)
.unwrap_or("")
.to_ascii_lowercase();
counter!("capture_ai_otel_requests_total", "format" => format).increment(1);
let auth_header = headers
.get("authorization")
.and_then(|v| v.to_str().ok())
.unwrap_or("");
if !auth_header.starts_with("Bearer ") {
let err = CaptureError::NoTokenError;
report_internal_error_metrics(err.to_metric_tag(), "otel_auth");
return Err(err.into_response());
}
let token = &auth_header[7..]; // Remove "Bearer " prefix
validate_token(token).map_err(|e| {
let err = CaptureError::from(e);
report_internal_error_metrics(err.to_metric_tag(), "otel_auth");
err.into_response()
})?;
if state.token_dropper.should_drop(token, "") {
report_dropped_events("token_dropper", 1);
return Ok(Json(json!({})));
}
let gateway_provenance = provenance::verify(
&headers,
state.ai_gateway_signing_secret.as_deref(),
token,
normalized_content_type,
&content_encoding,
&body,
state.timesource.current_time(),
);
let request = ingestion::parse_request(&body, &headers, OTEL_BODY_SIZE).map_err(|e| {
report_internal_error_metrics(e.to_metric_tag(), "otel_parsing");
// No parsed request to attribute from: the SDK identity lives inside the
// body we couldn't read.
emit_otel_parse_warning(
state.ingestion_warning_emitter.as_deref(),
&otel_request_context(token, OTEL_PATH, None),
&e,
format,
);
e.into_response()
})?;
let raw_span_count = count_spans(&request);
if raw_span_count == 0 {
counter!("capture_ai_otel_requests_success").increment(1);
return Ok(Json(json!({})));
}
// Cap raw spans before doing any expensive attribute conversion. The body
// size limit (4 MB) bounds the absolute maximum, but compact protobuf can
// pack many spans into that budget.
if raw_span_count > MAX_RAW_SPANS_PER_REQUEST {
let err = CaptureError::RequestParsingError(format!(
"Too many spans: {raw_span_count} exceeds limit of {MAX_RAW_SPANS_PER_REQUEST}"
));
report_internal_error_metrics(err.to_metric_tag(), "otel_validation");
emit_span_cap_warning(
state.ingestion_warning_emitter.as_deref(),
&otel_request_context(token, OTEL_PATH, Some(&request)),
SpanCapStage::Raw,
raw_span_count,
MAX_RAW_SPANS_PER_REQUEST,
);
return Err(err.into_response());
}
let received_at = Utc::now();
let request_fallback_distinct_id = identity::request_fallback_distinct_id();
let mut span_events = fan_out::expand_into_events(&request, &request_fallback_distinct_id);
provenance::apply(
&mut span_events,
gateway_provenance,
headers.contains_key(crate::gateway_provenance::SIGNATURE_HEADER),
headers
.get(crate::gateway_provenance::REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
);
let span_count = span_events.len();
let dropped_span_count = raw_span_count.saturating_sub(span_count);
Span::current().record("span_count", span_count);
if dropped_span_count > 0 {
counter!("capture_ai_otel_spans_filtered").increment(dropped_span_count as u64);
}
if span_count == 0 {
// Reached only with raw_span_count > 0 (the zero-span export returned
// above), so the customer sent spans and none of them landed. The OTLP
// contract has no way to say that in the response, which is why this
// 200 gets a warning and the mixed-batch case above does not.
emit_no_ai_spans_warning(
state.ingestion_warning_emitter.as_deref(),
&otel_request_context(token, OTEL_PATH, Some(&request)),
raw_span_count,
);
counter!("capture_ai_otel_requests_success").increment(1);
return Ok(Json(json!({})));
}
if span_count > MAX_SPANS_PER_REQUEST {
let err = CaptureError::RequestParsingError(format!(
"Too many AI spans: {span_count} exceeds limit of {MAX_SPANS_PER_REQUEST}"
));
report_internal_error_metrics(err.to_metric_tag(), "otel_validation");
emit_span_cap_warning(
state.ingestion_warning_emitter.as_deref(),
&otel_request_context(token, OTEL_PATH, Some(&request)),
SpanCapStage::Ai,
span_count,
MAX_SPANS_PER_REQUEST,
);
return Err(err.into_response());
}
counter!("capture_ai_otel_spans_accepted").increment(span_count as u64);
histogram!("capture_ai_otel_spans_per_request").record(span_count as f64);
let client_ip = ip
.map(|InsecureClientIp(addr)| addr.to_string())
.unwrap_or_else(|| "127.0.0.1".to_string());
let token = token.to_string();
// All-or-nothing quota check: reject the entire batch if any span is over quota
if let Err(outcome) = filtering::check_quota(
&state.quota_limiter,
&token,
&span_events,
gateway_provenance == provenance::Provenance::Verified,
)
.await
{
return match outcome {
filtering::QuotaOutcome::Dropped => Err(non_retryable_rejection("quota exceeded")),
filtering::QuotaOutcome::Error(e) => {
report_internal_error_metrics(e.to_metric_tag(), "otel_quota");
Err(e.into_response())
}
};
}
let restrictions = match &state.event_restriction_service {
Some(service) => {
let now_ts = state.timesource.current_time().timestamp();
filtering::check_restrictions(service, &token, now_ts, &span_events)
.await
.map_err(|_| non_retryable_rejection("event restricted"))?
}
None => Default::default(),
};
let mut processed_events =
filtering::build_events(span_events, &token, &client_ip, received_at, &restrictions)
.map_err(|e| {
report_internal_error_metrics(e.to_metric_tag(), "otel_processing");
e.into_response()
})?;
// Shed spans past the deployment's per-event ceiling rather than refusing
// the export.
//
// Every other AI path answers an oversize event with an error: the legacy
// batch path 413s the request and v1 marks the single event dropped. OTEL
// diverges because a collector retries a rejected export, so a span that
// can never fit would stall every span queued behind it, indefinitely. The
// span cap on this endpoint sheds spans for the same reason.
//
// The cost is that the loss cannot be seen in the response, so it is
// reported twice: on `capture_events_dropped_total` for us, and as a
// `MessageSizeTooLarge` ingestion warning for whoever owns the project.
let before = processed_events.len();
processed_events
.retain(|e| !exceeds_max_ai_event_bytes(e.event.data.len(), state.ai_max_event_bytes));
let dropped = before - processed_events.len();
if dropped > 0 {
report_dropped_events("ai_event_too_big", dropped as u64);
emit_span_too_big_warning(
state.ingestion_warning_emitter.as_deref(),
&otel_request_context(&token, OTEL_PATH, Some(&request)),
dropped,
state.ai_max_event_bytes,
);
}
// Charge the AI lane's per-project byte budget, shedding the spans that
// take the project past it. This endpoint builds its events at the handler
// and reaches the sink through neither analytics pipeline, so without this
// call a sender could spend an unbounded number of bytes here while the
// same bytes on `/i/v0/ai/batch` are capped.
//
// Shedding rather than refusing, for the reason the size ceiling above
// sheds: a collector retries a rejected export, so refusing would stall
// every span behind the ones over budget. Unlike the size ceiling this
// raises no ingestion warning — a rate drop is ops-imposed, and capture
// surfaces those through billing and ops channels rather than the
// customer-facing warnings table, which `warning_for_capture_error` pins.
//
// It runs after the ceiling so an event too big to publish spends none of
// the budget, matching the order both analytics pipelines use.
drop_ai_byte_limited(&mut processed_events, state.ai_byte_rate_limiter.as_ref()).await;
// Apply the in-process OverflowLimiter governor to every AnalyticsMain
// span in the batch before handing off to the sink. OTEL bypasses
// `events::analytics::process_events`, so this call is what preserves
// OverflowLimiter parity on `capture-ai-*` deploys (where
// `OVERFLOW_ENABLED=true`). Per-span key evaluation matches the analytics
// batch path: spans with different `token:distinct_id` keys can land
// with different `overflow_reason` stamps in the same batch.
stamp_overflow_reason(
&mut processed_events,
state.overflow_limiter.as_ref(),
state.ai_events_overflow_limiter.as_ref(),
);
// Count what the sink was handed, not what arrived: `span_count` predates
// the size ceiling and the byte budget, either of which may have shed spans.
let ingested = processed_events.len() as u64;
histogram!("capture_event_batch_size").record(processed_events.len() as f64);
state.outputs.publish(processed_events).await.map_err(|e| {
report_internal_error_metrics(e.to_metric_tag(), "otel_sink");
warn!("Failed to send OTel events to Kafka: {:?}", e);
e.into_response()
})?;
counter!("capture_ai_otel_events_ingested").increment(ingested);
counter!("capture_ai_otel_requests_success").increment(1);
debug!(
"OTEL endpoint request processed successfully: {} spans",
span_count
);
// Return empty JSON object per OTLP spec
Ok(Json(json!({})))
}
pub async fn options() -> Result<CaptureResponse, CaptureError> {
Ok(CaptureResponse {
status: CaptureResponseCode::Ok,
quota_limited: None,
})
}