:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
#[path = "common/integration_utils.rs"]
mod integration_utils;
use async_trait::async_trait;
use axum_test_helper::TestClient;
use capture::api::CaptureError;
use capture::config::CaptureMode;
use capture::event_restrictions::{
EventRestrictionService, Pipeline, Restriction, RestrictionFilters, RestrictionManager,
RestrictionScope, RestrictionType,
};
use capture::global_rate_limiter::GlobalRateLimiter;
use capture::outputs::{OutputRegistry, PreparedEvent, PublishEvents, PublishPrepared};
use capture::quota_limiters::{is_llm_event, CaptureQuotaLimiter, EventInfo};
use capture::router::router;
use capture::sinks::sink::SinkResult;
use capture::time::TimeSource;
use capture::v0_request::{DataType, OverflowReason, ProcessedEvent};
use chrono::{DateTime, Utc};
use common_ingestion_warnings::test_support::CollectingEmitter;
use common_ingestion_warnings::{WarningEmitter, WarningType, CAPTURE_AI_OTEL};
use common_redis::MockRedisClient;
use hmac::{Hmac, Mac};
use integration_utils::{test_lifecycle_handlers, DEFAULT_TEST_TIME};
use limiters::overflow::OverflowLimiter;
use limiters::redis::{QuotaResource, QUOTA_LIMITER_CACHE_KEY};
use limiters::token_dropper::TokenDropper;
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, KeyValue};
use opentelemetry_proto::tonic::resource::v1::Resource;
use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span};
use prost::Message;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::collections::HashSet;
use std::io::Write;
use std::num::NonZeroU32;
use std::sync::Arc;
use std::time::Duration;
use integration_utils::DEFAULT_CONFIG;
const GATEWAY_REQUEST_ID: &str = "otel-request-1";
struct FixedTime {
pub time: DateTime<Utc>,
}
impl TimeSource for FixedTime {
fn current_time(&self) -> DateTime<Utc> {
self.time
}
}
#[derive(Clone)]
struct CapturingSink {
events: Arc<tokio::sync::Mutex<Vec<ProcessedEvent>>>,
}
impl CapturingSink {
fn new() -> Self {
Self {
events: Arc::new(tokio::sync::Mutex::new(Vec::new())),
}
}
async fn get_events(&self) -> Vec<ProcessedEvent> {
self.events.lock().await.clone()
}
}
#[async_trait]
impl PublishEvents for CapturingSink {
async fn publish_events(&self, events: Vec<ProcessedEvent>) -> Result<(), CaptureError> {
self.events.lock().await.extend(events);
Ok(())
}
}
#[async_trait]
impl PublishPrepared for CapturingSink {
async fn publish_prepared(&self, _events: Vec<PreparedEvent>) -> Vec<SinkResult> {
unreachable!("v0 endpoints publish events")
}
}
const TOKEN: &str = "phc_VXRzc3poSG9GZm1JenRianJ6TTJFZGh4OWY2QXzx9f3";
fn make_kv(key: &str, value: any_value::Value) -> KeyValue {
KeyValue {
key: key.to_string(),
value: Some(AnyValue { value: Some(value) }),
}
}
fn make_span(
trace_id: Vec<u8>,
span_id: Vec<u8>,
parent_span_id: Vec<u8>,
start_time_nanos: u64,
attributes: Vec<KeyValue>,
) -> Span {
Span {
trace_id,
span_id,
parent_span_id,
start_time_unix_nano: start_time_nanos,
attributes,
..Default::default()
}
}
fn make_irrelevant_http_span(trace_id: Vec<u8>, span_id: Vec<u8>) -> Span {
make_span(
trace_id,
span_id,
vec![],
1_704_067_200_000_000_000,
vec![
make_kv(
"http.request.method",
any_value::Value::StringValue("POST".to_string()),
),
make_kv(
"url.full",
any_value::Value::StringValue("https://example.com/api".to_string()),
),
],
)
}
#[derive(Default)]
struct TestClientOptions {
redis: Option<Arc<MockRedisClient>>,
event_restriction_service: Option<EventRestrictionService>,
quota_limiter: Option<CaptureQuotaLimiter>,
ai_gateway_signing_secret: Option<String>,
// `None` matches deploys without both `OVERFLOW_ENABLED` and `AI_EVENTS_OVERFLOW_ENABLED`.
ai_events_overflow_limiter: Option<Arc<OverflowLimiter>>,
// Opt-in warnings emitter. `None` (default) matches deploys without
// `CAPTURE_INGESTION_WARNINGS_ENABLED` and exercises the no-op branch of
// every emit site.
ingestion_warning_emitter: Option<Arc<dyn WarningEmitter>>,
// `None` uses 960KB (983_040).
ai_max_event_bytes: Option<u64>,
// Opt-in AI byte budget. `None` (default) matches deploys with
// `AI_BYTE_LIMIT_PER_SECOND=0`, where setup builds no limiter.
ai_byte_rate_limiter: Option<Arc<GlobalRateLimiter>>,
}
fn make_test_client(sink: &CapturingSink) -> TestClient {
make_test_client_with_options(sink, TestClientOptions::default())
}
fn make_test_client_with_options(sink: &CapturingSink, options: TestClientOptions) -> TestClient {
let (readiness, liveness, _monitor) = test_lifecycle_handlers();
let timesource = FixedTime {
time: DateTime::parse_from_rfc3339(DEFAULT_TEST_TIME)
.expect("Invalid fixed time format")
.with_timezone(&Utc),
};
let redis = options
.redis
.unwrap_or_else(|| Arc::new(MockRedisClient::new()));
let mut cfg = DEFAULT_CONFIG.clone();
cfg.capture_mode = CaptureMode::Ai;
let quota_limiter = options.quota_limiter.unwrap_or_else(|| {
CaptureQuotaLimiter::new(&cfg, redis.clone(), Duration::from_secs(60 * 60 * 24 * 7))
.add_scoped_limiter(QuotaResource::LLMEvents, is_llm_event)
});
let app = router(
timesource,
readiness,
liveness,
Arc::new(OutputRegistry::single(sink.clone())),
redis,
None, // global_rate_limiter_token_distinctid
quota_limiter,
TokenDropper::default(),
options.event_restriction_service,
None, // recorder_handle
CaptureMode::Ai,
None, // concurrency_limit
25 * 1024 * 1024, // event_payload_size_limit
false, // enable_historical_rerouting
1_i64, // historical_rerouting_threshold_days
false, // is_mirror_deploy
0.0_f32, // verbose_sample_percent
26_214_400, // ai_max_sum_of_parts_bytes
options.ai_max_event_bytes.unwrap_or(983_040), // ai_max_event_bytes
None, // body_chunk_read_timeout_ms
256, // body_read_chunk_size_kb
10 * 1024 * 1024, // capture_v1_max_compressed_body_bytes
50 * 1024 * 1024, // capture_v1_max_decompressed_body_bytes
None, // overflow_limiter
options.ai_events_overflow_limiter,
options.ai_byte_rate_limiter,
None, // replay_overflow_limiter
8, // capture_v1_scatter_gather_min_batch
options.ai_gateway_signing_secret,
options.ingestion_warning_emitter, // ingestion_warning_emitter
);
TestClient::new(app)
}
/// A client whose emit sites are live, plus the emitter to assert against.
fn make_test_client_collecting_warnings(
sink: &CapturingSink,
) -> (TestClient, Arc<CollectingEmitter>) {
let emitter = Arc::new(CollectingEmitter::default());
let client = make_test_client_with_options(
sink,
TestClientOptions {
ingestion_warning_emitter: Some(emitter.clone()),
..Default::default()
},
);
(client, emitter)
}
const ENDPOINT: &str = "/i/v0/ai/otel";
async fn send_request(sink: &CapturingSink, request: &ExportTraceServiceRequest) -> u16 {
let client = make_test_client(sink);
let body = request.encode_to_vec();
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(body)
.send()
.await;
resp.status().as_u16()
}
async fn send_request_with_client(client: &TestClient, request: &ExportTraceServiceRequest) -> u16 {
let body = request.encode_to_vec();
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(body)
.send()
.await;
resp.status().as_u16()
}
async fn send_signed_request_with_client(
client: &TestClient,
request: &ExportTraceServiceRequest,
secret: &str,
) -> u16 {
let body = request.encode_to_vec();
let signature = sign_gateway_body(
secret,
"application/x-protobuf",
"",
&body,
DEFAULT_TEST_TIME,
);
send_signed_body_with_client(client, body, "application/x-protobuf", signature).await
}
async fn send_signed_body_with_client(
client: &TestClient,
body: Vec<u8>,
content_type: &str,
signature: String,
) -> u16 {
let resp = client
.post(ENDPOINT)
.header("Content-Type", content_type)
.header("Authorization", format!("Bearer {}", TOKEN))
.header("PostHog-Ai-Gateway-Signature", signature)
.header("PostHog-Ai-Gateway-Signed-At", DEFAULT_TEST_TIME)
.header("PostHog-Ai-Gateway-Request-Id", GATEWAY_REQUEST_ID)
.body(body)
.send()
.await;
resp.status().as_u16()
}
fn sign_gateway_body(
secret: &str,
content_type: &str,
content_encoding: &str,
body: &[u8],
signed_at: &str,
) -> String {
let body_digest = hex::encode(Sha256::digest(body));
let fields = [
TOKEN,
"otel-v1",
content_type,
content_encoding,
body_digest.as_str(),
GATEWAY_REQUEST_ID,
signed_at,
];
let mut message = Vec::new();
for field in fields {
message.extend_from_slice(&(field.len() as u32).to_be_bytes());
message.extend_from_slice(field.as_bytes());
}
let mut mac = Hmac::<Sha256>::new_from_slice(secret.as_bytes()).unwrap();
mac.update(&message);
hex::encode(mac.finalize().into_bytes())
}
fn parse_event_data(event: &ProcessedEvent) -> serde_json::Value {
serde_json::from_str(&event.event.data).expect("event data is valid JSON")
}
fn make_single_span_request() -> ExportTraceServiceRequest {
ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-1".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(
vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
vec![1, 2, 3, 4, 5, 6, 7, 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
)],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
}
}
fn make_two_span_request() -> ExportTraceServiceRequest {
ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-1".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_span(
vec![1; 16],
vec![1; 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
make_span(
vec![1; 16],
vec![2; 8],
vec![1; 8],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("embeddings".to_string()),
)],
),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
}
}
async fn make_restriction_service(restrictions: Vec<Restriction>) -> EventRestrictionService {
// OTel runs in the AI capture deployment, so restrictions must be indexed
// under Pipeline::Ai — that's the pipeline `otel/filtering.rs` looks up
// with for every span.
let service = EventRestrictionService::new(vec![Pipeline::Ai], Duration::from_secs(300));
let mut manager = RestrictionManager::new();
manager.insert_restrictions(Pipeline::Ai, TOKEN, restrictions);
service.update(manager).await;
service
}
#[tokio::test]
async fn test_single_span_produces_one_event() {
let sink = CapturingSink::new();
let request = make_single_span_request();
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 1);
let event = &events[0];
assert_eq!(event.event.token, TOKEN);
assert_eq!(event.event.event, "$ai_generation");
assert_eq!(event.event.distinct_id, "user-1");
assert_eq!(event.metadata.data_type, DataType::AiEvents);
assert_eq!(event.metadata.event_name, "$ai_generation");
let data = parse_event_data(event);
assert_eq!(
data["properties"]["$ai_trace_id"],
"0102030405060708090a0b0c0d0e0f10"
);
assert_eq!(data["properties"]["$ai_span_id"], "0102030405060708");
assert_eq!(data["properties"]["$ai_ingestion_source"], "otel");
}
#[tokio::test]
async fn test_verified_gateway_batch_stamps_trusted_provenance() {
const SECRET: &str = "test-signing-secret";
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let status =
send_signed_request_with_client(&client, &make_single_span_request(), SECRET).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
let data = parse_event_data(&events[0]);
assert_eq!(data["properties"]["$ai_gateway_verified"], true);
assert_eq!(data["properties"]["$ai_gateway_relay"], true);
}
#[tokio::test]
async fn test_verified_gzip_gateway_batch_stamps_trusted_provenance() {
const SECRET: &str = "test-signing-secret";
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let body = make_single_span_request().encode_to_vec();
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder.write_all(&body).unwrap();
let body = encoder.finish().unwrap();
let signature = sign_gateway_body(
SECRET,
"application/x-protobuf",
"gzip",
&body,
DEFAULT_TEST_TIME,
);
let response = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Content-Encoding", "gzip")
.header("Authorization", format!("Bearer {}", TOKEN))
.header("PostHog-Ai-Gateway-Signature", signature)
.header("PostHog-Ai-Gateway-Signed-At", DEFAULT_TEST_TIME)
.header("PostHog-Ai-Gateway-Request-Id", GATEWAY_REQUEST_ID)
.body(body)
.send()
.await;
assert_eq!(response.status().as_u16(), 200);
let events = sink.get_events().await;
let data = parse_event_data(&events[0]);
assert_eq!(data["properties"]["$ai_gateway_verified"], true);
assert_eq!(data["properties"]["$ai_gateway_relay"], true);
}
#[tokio::test]
async fn test_signed_json_batch_stamps_trusted_provenance() {
const SECRET: &str = "test-signing-secret";
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let body = serde_json::to_vec(&make_single_span_request()).unwrap();
let signature = sign_gateway_body(SECRET, "application/json", "", &body, DEFAULT_TEST_TIME);
let status = send_signed_body_with_client(&client, body, "application/json", signature).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
let data = parse_event_data(&events[0]);
assert_eq!(data["properties"]["$ai_gateway_verified"], true);
assert_eq!(data["properties"]["$ai_gateway_relay"], true);
}
#[tokio::test]
async fn test_signed_json_body_mismatch_rejects_trusted_provenance() {
const SECRET: &str = "test-signing-secret";
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let signed_body = serde_json::to_vec(&make_single_span_request()).unwrap();
let signature = sign_gateway_body(
SECRET,
"application/json",
"",
&signed_body,
DEFAULT_TEST_TIME,
);
let mut request = make_single_span_request();
request.resource_spans[0].schema_url = "tampered".to_string();
let body = serde_json::to_vec(&request).unwrap();
let status = send_signed_body_with_client(&client, body, "application/json", signature).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
let data = parse_event_data(&events[0]);
let properties = data["properties"].as_object().unwrap();
assert!(!properties.contains_key("$ai_gateway_verified"));
assert!(!properties.contains_key("$ai_gateway_relay"));
}
#[tokio::test]
async fn test_unsigned_batch_cannot_forge_gateway_provenance() {
let sink = CapturingSink::new();
let mut request = make_single_span_request();
let attributes = &mut request.resource_spans[0].scope_spans[0].spans[0].attributes;
attributes.extend([
make_kv("$ai_gateway_verified", any_value::Value::BoolValue(true)),
make_kv("$ai_gateway_relay", any_value::Value::BoolValue(true)),
make_kv(
"$ai_gateway_request_id",
any_value::Value::StringValue("forged".to_string()),
),
]);
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
let data = parse_event_data(&events[0]);
let properties = data["properties"].as_object().unwrap();
assert!(!properties.contains_key("$ai_gateway_verified"));
assert!(!properties.contains_key("$ai_gateway_relay"));
assert!(!properties.contains_key("$ai_gateway_request_id"));
}
#[tokio::test]
async fn test_multiple_spans_produce_multiple_events() {
let sink = CapturingSink::new();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-2".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_span(
vec![1; 16],
vec![1; 8],
vec![],
0,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
make_span(
vec![1; 16],
vec![2; 8],
vec![1; 8],
0,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("embeddings".to_string()),
)],
),
make_span(
vec![1; 16],
vec![3; 8],
vec![],
0,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("unknown_op".to_string()),
)],
),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 3);
assert_eq!(events[0].event.event, "$ai_generation");
assert_eq!(events[1].event.event, "$ai_embedding");
assert_eq!(events[2].event.event, "$ai_span");
let data1 = parse_event_data(&events[1]);
assert_eq!(data1["properties"]["$ai_parent_id"], "0101010101010101");
}
#[tokio::test]
async fn test_span_attributes_and_resource_attributes_passthrough() {
let sink = CapturingSink::new();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![
make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-3".to_string()),
),
make_kv(
"service.name",
any_value::Value::StringValue("my-service".to_string()),
),
],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(
vec![0xAB; 16],
vec![0xCD; 8],
vec![],
0,
vec![
make_kv(
"gen_ai.request.model",
any_value::Value::StringValue("gpt-4".to_string()),
),
make_kv("gen_ai.usage.input_tokens", any_value::Value::IntValue(100)),
make_kv(
"custom.attr",
any_value::Value::StringValue("custom-val".to_string()),
),
],
)],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 1);
let data = parse_event_data(&events[0]);
let props = &data["properties"];
assert_eq!(props["gen_ai.request.model"], "gpt-4");
assert_eq!(props["gen_ai.usage.input_tokens"], 100);
assert_eq!(props["custom.attr"], "custom-val");
assert_eq!(props["service.name"], "my-service");
assert_eq!(props["$ai_ingestion_source"], "otel");
}
#[tokio::test]
async fn test_multiple_resource_spans() {
let sink = CapturingSink::new();
let request = ExportTraceServiceRequest {
resource_spans: vec![
ResourceSpans {
resource: Some(Resource {
attributes: vec![
make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-4".to_string()),
),
make_kv(
"service.name",
any_value::Value::StringValue("svc-a".to_string()),
),
],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(
vec![1; 16],
vec![1; 8],
vec![],
0,
vec![make_kv(
"gen_ai.request.model",
any_value::Value::StringValue("gpt-4o-mini".to_string()),
)],
)],
schema_url: String::new(),
}],
schema_url: String::new(),
},
ResourceSpans {
resource: Some(Resource {
attributes: vec![
make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-4".to_string()),
),
make_kv(
"service.name",
any_value::Value::StringValue("svc-b".to_string()),
),
],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(
vec![2; 16],
vec![2; 8],
vec![],
0,
vec![make_kv(
"gen_ai.request.model",
any_value::Value::StringValue("claude-3-5-sonnet".to_string()),
)],
)],
schema_url: String::new(),
}],
schema_url: String::new(),
},
],
};
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 2);
let data0 = parse_event_data(&events[0]);
let data1 = parse_event_data(&events[1]);
assert_eq!(data0["properties"]["service.name"], "svc-a");
assert_eq!(data1["properties"]["service.name"], "svc-b");
}
#[tokio::test]
async fn test_irrelevant_http_spans_are_ignored() {
let sink = CapturingSink::new();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-http".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_irrelevant_http_span(vec![9; 16], vec![8; 8])],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert!(events.is_empty());
}
#[tokio::test]
async fn test_mixed_requests_only_emit_relevant_ai_spans() {
let sink = CapturingSink::new();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-mixed".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_irrelevant_http_span(vec![3; 16], vec![1; 8]),
make_span(
vec![3; 16],
vec![2; 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 1);
assert_eq!(events[0].event.event, "$ai_generation");
}
/// A real limiter whose local cache admits a key's first charge and limits
/// every one after it, so a single multi-span export shows both outcomes.
///
/// A threshold of `0` is what makes the second charge exceed the window with no
/// Redis round trip and no clock, the same seam
/// `integration_person_processing_matrix` uses for the event limiter. The tick
/// is parked well past the request so no background sync can race it.
fn always_limits_after_the_first_span() -> Arc<GlobalRateLimiter> {
let mut cfg = DEFAULT_CONFIG.clone();
cfg.capture_mode = CaptureMode::Ai;
cfg.ai_byte_limit_per_second = 0;
cfg.global_rate_limit_tick_interval_ms = 600_000;
Arc::new(
GlobalRateLimiter::new_ai_bytes(&cfg, vec![Arc::new(MockRedisClient::new())])
.expect("failed to build the AI byte limiter"),
)
}
/// The byte budget reaches this endpoint. It builds its events at the handler
/// and never enters either analytics pipeline, so the charge has to be wired
/// here or a sender could spend unbounded bytes on `/i/v0/ai/otel` while the
/// same bytes are capped on the batch paths.
///
/// Over-budget spans are shed rather than the export refused, matching the size
/// ceiling and for the same reason: a collector retries a rejected export.
#[tokio::test]
async fn over_budget_spans_are_shed_while_the_export_succeeds() {
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_byte_rate_limiter: Some(always_limits_after_the_first_span()),
..Default::default()
},
);
let status = send_request_with_client(&client, &make_two_span_request()).await;
assert_eq!(status, 200, "the export must still be accepted");
let events = sink.get_events().await;
assert_eq!(
events.len(),
1,
"the first span is admitted on a cold key; the second is over budget"
);
assert_eq!(events[0].event.event, "$ai_generation");
}
/// Without a limiter configured, both spans publish — so the test above is
/// pinning the budget rather than some other filter in the handler.
#[tokio::test]
async fn both_spans_publish_when_no_byte_budget_is_configured() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let status = send_request_with_client(&client, &make_two_span_request()).await;
assert_eq!(status, 200);
assert_eq!(sink.get_events().await.len(), 2);
}
/// An oversized span is shed and the export still succeeds, so a collector is
/// never made to retry a span that can never fit. The warning is the only
/// feedback channel for that, which is why it is asserted alongside the drop:
/// without it the customer sees a 200 and silently missing spans.
#[tokio::test]
async fn oversized_spans_are_shed_and_warn_while_the_export_succeeds() {
let sink = CapturingSink::new();
let emitter = Arc::new(CollectingEmitter::default());
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ingestion_warning_emitter: Some(emitter.clone()),
ai_max_event_bytes: Some(64),
..Default::default()
},
);
// A 64-byte ceiling is under the serialized size of even a minimal AI span.
let status = send_request_with_client(&client, &make_single_span_request()).await;
assert_eq!(status, 200, "the export must still be accepted");
assert!(
sink.get_events().await.is_empty(),
"the oversized span must not reach the sink"
);
let emitted = emitter.emitted();
assert_eq!(emitted.len(), 1);
assert_eq!(emitted[0].warning, WarningType::MessageSizeTooLarge);
assert_eq!(emitted[0].source, CAPTURE_AI_OTEL);
assert_eq!(
emitted[0].extra_details.get("droppedSpans"),
Some(&json!(1))
);
}
// The warning is the only feedback channel for this outcome: the OTLP contract
// has no way to say "accepted, ingested nothing", so the exporter sees a 200.
// Also proves the emit site is wired and that SDK attribution survives the trip
// from resource attributes into the warning, neither of which a unit test on the
// helper can catch.
#[tokio::test]
async fn all_spans_filtered_warns_and_still_returns_200() {
let sink = CapturingSink::new();
let (client, emitter) = make_test_client_collecting_warnings(&sink);
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![
make_kv(
"telemetry.sdk.name",
any_value::Value::StringValue("opentelemetry-js".to_string()),
),
make_kv(
"telemetry.sdk.version",
any_value::Value::StringValue("2.0.0".to_string()),
),
],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_irrelevant_http_span(vec![9; 16], vec![8; 8]),
make_irrelevant_http_span(vec![9; 16], vec![7; 8]),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 200);
assert!(sink.get_events().await.is_empty());
let emitted = emitter.emitted();
assert_eq!(emitted.len(), 1);
assert_eq!(emitted[0].warning, WarningType::NoAiSpansIngested);
assert_eq!(emitted[0].source, CAPTURE_AI_OTEL);
assert_eq!(emitted[0].token, TOKEN);
assert_eq!(emitted[0].count, 2);
assert_eq!(
emitted[0].extra_details.get("rawSpanCount"),
Some(&json!(2))
);
assert_eq!(
emitted[0].extra_details.get("lib"),
Some(&json!("opentelemetry-js"))
);
assert_eq!(
emitted[0].extra_details.get("libVersion"),
Some(&json!("2.0.0"))
);
assert_eq!(
emitted[0].extra_details.get("path"),
Some(&json!("/i/v0/ai/otel"))
);
}
// Mixed batches are documented-expected (a framework exporting HTTP and AI spans
// together), so warning on them would fire on correct usage. Pins that choice
// against a future refactor that starts warning whenever anything is filtered.
#[tokio::test]
async fn mixed_batch_ingests_ai_spans_without_warning() {
let sink = CapturingSink::new();
let (client, emitter) = make_test_client_collecting_warnings(&sink);
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: None,
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_irrelevant_http_span(vec![3; 16], vec![1; 8]),
make_span(
vec![3; 16],
vec![2; 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 200);
assert_eq!(sink.get_events().await.len(), 1);
assert!(emitter.emitted().is_empty());
}
#[tokio::test]
async fn too_many_ai_spans_warns_with_the_ai_stage() {
let sink = CapturingSink::new();
let (client, emitter) = make_test_client_collecting_warnings(&sink);
let over_cap = 101;
let spans = (0..over_cap)
.map(|i| {
make_span(
vec![4; 16],
vec![i as u8; 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
)
})
.collect();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: None,
scope_spans: vec![ScopeSpans {
scope: None,
spans,
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 400);
assert!(sink.get_events().await.is_empty());
let emitted = emitter.emitted();
assert_eq!(emitted.len(), 1);
assert_eq!(emitted[0].warning, WarningType::InvalidAiPayload);
assert_eq!(emitted[0].count, over_cap as u64);
assert_eq!(emitted[0].extra_details.get("stage"), Some(&json!("ai")));
assert_eq!(
emitted[0].extra_details.get("spanCount"),
Some(&json!(over_cap))
);
assert_eq!(emitted[0].extra_details.get("limit"), Some(&json!(100)));
}
// Attribution has to fall back to unknown here: the SDK identity lives inside
// the body that failed to parse.
#[tokio::test]
async fn unparseable_body_warns_with_unknown_attribution() {
let sink = CapturingSink::new();
let (client, emitter) = make_test_client_collecting_warnings(&sink);
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(vec![0xff, 0xff, 0xff, 0xff])
.send()
.await;
assert_eq!(resp.status().as_u16(), 400);
let emitted = emitter.emitted();
assert_eq!(emitted.len(), 1);
assert_eq!(emitted[0].warning, WarningType::InvalidAiPayload);
assert_eq!(emitted[0].count, 1);
assert_eq!(
emitted[0].extra_details.get("format"),
Some(&json!("protobuf"))
);
assert_eq!(emitted[0].extra_details.get("lib"), Some(&json!("unknown")));
}
#[tokio::test]
async fn test_empty_body_returns_400() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(vec![])
.send()
.await;
assert_eq!(resp.status().as_u16(), 400);
}
#[tokio::test]
async fn test_missing_auth_returns_401() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: None,
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(vec![0; 16], vec![0; 8], vec![], 0, vec![])],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.body(request.encode_to_vec())
.send()
.await;
assert_eq!(resp.status().as_u16(), 401);
}
#[tokio::test]
async fn test_invalid_token_returns_401() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: None,
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(vec![0; 16], vec![0; 8], vec![], 0, vec![])],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", "Bearer phx_personal_api_key_not_allowed")
.body(request.encode_to_vec())
.send()
.await;
assert_eq!(resp.status().as_u16(), 401);
}
#[tokio::test]
async fn test_unsupported_content_encoding_returns_400() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: None,
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![make_span(vec![0; 16], vec![0; 8], vec![], 0, vec![])],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Content-Encoding", "deflate")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(request.encode_to_vec())
.send()
.await;
assert_eq!(resp.status().as_u16(), 400);
}
#[tokio::test]
async fn test_too_many_spans_returns_400() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
let spans: Vec<Span> = (0..101)
.map(|i| {
make_span(
vec![0; 16],
vec![i as u8; 8],
vec![],
0,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
)
})
.collect();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-limit".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans,
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(request.encode_to_vec())
.send()
.await;
assert_eq!(resp.status().as_u16(), 400);
}
#[tokio::test]
async fn test_too_many_raw_spans_returns_400() {
let sink = CapturingSink::new();
let client = make_test_client(&sink);
// 1001 non-AI spans exceeds the MAX_RAW_SPANS_PER_REQUEST limit of 1000.
let spans: Vec<Span> = (0..1001u16)
.map(|i| {
let id_bytes: Vec<u8> = i.to_be_bytes().iter().chain(&[0u8; 6]).copied().collect();
make_span(
vec![0; 16],
id_bytes,
vec![],
0,
vec![make_kv(
"http.request.method",
any_value::Value::StringValue("GET".to_string()),
)],
)
})
.collect();
let request = ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-raw-limit".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans,
schema_url: String::new(),
}],
schema_url: String::new(),
}],
};
let resp = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.body(request.encode_to_vec())
.send()
.await;
assert_eq!(resp.status().as_u16(), 400);
}
// ----------------------------------------------------------------------------
// Quota Limiter Tests
// ----------------------------------------------------------------------------
#[tokio::test]
async fn test_quota_limit_exceeded_returns_400_with_no_events() {
let llm_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::LLMEvents.as_str()
);
let redis =
Arc::new(MockRedisClient::new().zrangebyscore_ret(&llm_key, vec![TOKEN.to_string()]));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
..Default::default()
},
);
let request = make_single_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 400);
let events = sink.get_events().await;
assert!(events.is_empty());
}
#[tokio::test]
async fn test_verified_gateway_batch_bypasses_scoped_llm_quota() {
const SECRET: &str = "test-signing-secret";
let llm_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::LLMEvents.as_str()
);
let redis =
Arc::new(MockRedisClient::new().zrangebyscore_ret(&llm_key, vec![TOKEN.to_string()]));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let status =
send_signed_request_with_client(&client, &make_single_span_request(), SECRET).await;
assert_eq!(status, 200);
assert_eq!(sink.get_events().await.len(), 1);
}
#[tokio::test]
async fn test_invalid_gateway_signature_does_not_bypass_scoped_llm_quota() {
const SECRET: &str = "test-signing-secret";
let llm_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::LLMEvents.as_str()
);
let redis =
Arc::new(MockRedisClient::new().zrangebyscore_ret(&llm_key, vec![TOKEN.to_string()]));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let body = make_single_span_request().encode_to_vec();
let response = client
.post(ENDPOINT)
.header("Content-Type", "application/x-protobuf")
.header("Authorization", format!("Bearer {}", TOKEN))
.header("PostHog-Ai-Gateway-Signature", "00")
.header("PostHog-Ai-Gateway-Signed-At", DEFAULT_TEST_TIME)
.body(body)
.send()
.await;
assert_eq!(response.status().as_u16(), 400);
assert!(sink.get_events().await.is_empty());
}
#[tokio::test]
async fn test_verified_gateway_batch_matches_direct_ai_global_quota_behavior() {
const SECRET: &str = "test-signing-secret";
let llm_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::LLMEvents.as_str()
);
let global_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::Events.as_str()
);
let redis = Arc::new(
MockRedisClient::new()
.zrangebyscore_ret(&llm_key, vec![TOKEN.to_string()])
.zrangebyscore_ret(&global_key, vec![TOKEN.to_string()]),
);
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
ai_gateway_signing_secret: Some(SECRET.to_string()),
..Default::default()
},
);
let status =
send_signed_request_with_client(&client, &make_single_span_request(), SECRET).await;
assert_eq!(status, 400);
assert!(sink.get_events().await.is_empty());
}
#[tokio::test]
async fn test_global_quota_exceeded_retains_scoped_llm_events() {
// When the global quota is exceeded but the scoped LLM limiter is not, the
// CaptureQuotaLimiter retains LLM events. Since all OTel spans are $ai_*
// events, they all get retained — no partial drop occurs and the batch
// goes through normally.
let global_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::Events.as_str()
);
let redis =
Arc::new(MockRedisClient::new().zrangebyscore_ret(&global_key, vec![TOKEN.to_string()]));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
..Default::default()
},
);
let request = make_single_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 1);
}
#[tokio::test]
async fn test_both_global_and_scoped_quota_exceeded_returns_400() {
let llm_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::LLMEvents.as_str()
);
let global_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::Events.as_str()
);
let redis = Arc::new(
MockRedisClient::new()
.zrangebyscore_ret(&llm_key, vec![TOKEN.to_string()])
.zrangebyscore_ret(&global_key, vec![TOKEN.to_string()]),
);
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
..Default::default()
},
);
let request = make_single_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 400);
let events = sink.get_events().await;
assert!(events.is_empty());
}
#[tokio::test]
async fn test_partial_quota_drop_rejects_entire_batch() {
// Use a custom scoped limiter that only matches $ai_generation (not $ai_embedding).
// When this limiter is exceeded, $ai_generation spans are dropped but $ai_embedding
// spans are retained → partial drop → all-or-nothing rejection returns 400.
let exceptions_key = format!(
"{}{}",
QUOTA_LIMITER_CACHE_KEY,
QuotaResource::Exceptions.as_str()
);
let redis = Arc::new(
MockRedisClient::new().zrangebyscore_ret(&exceptions_key, vec![TOKEN.to_string()]),
);
let cfg = DEFAULT_CONFIG.clone();
let quota_limiter =
CaptureQuotaLimiter::new(&cfg, redis.clone(), Duration::from_secs(60 * 60 * 24 * 7))
.add_scoped_limiter(QuotaResource::Exceptions, |info: EventInfo| {
info.name == "$ai_generation"
});
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
redis: Some(redis),
quota_limiter: Some(quota_limiter),
..Default::default()
},
);
// Send two spans: one $ai_generation, one $ai_embedding
let request = make_two_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 400);
let events = sink.get_events().await;
assert!(events.is_empty());
}
// ----------------------------------------------------------------------------
// Event Restriction Tests
// ----------------------------------------------------------------------------
#[tokio::test]
async fn test_restriction_types() {
struct Case {
name: &'static str,
restriction_type: RestrictionType,
args: Option<serde_json::Value>,
expected_status: u16,
check: fn(&[ProcessedEvent]) -> bool,
}
let cases = [
Case {
name: "drop",
restriction_type: RestrictionType::DropEvent,
args: None,
expected_status: 400,
check: |events| events.is_empty(),
},
Case {
name: "force_overflow",
restriction_type: RestrictionType::ForceOverflow,
args: None,
expected_status: 200,
check: |events| events.len() == 1 && events[0].metadata.force_overflow,
},
Case {
name: "skip_person_processing",
restriction_type: RestrictionType::SkipPersonProcessing,
args: None,
expected_status: 200,
check: |events| events.len() == 1 && events[0].metadata.skip_person_processing,
},
Case {
name: "redirect_to_dlq",
restriction_type: RestrictionType::RedirectToDlq,
args: None,
expected_status: 200,
check: |events| events.len() == 1 && events[0].metadata.redirect_to_dlq,
},
Case {
name: "redirect_to_topic",
restriction_type: RestrictionType::RedirectToTopic,
args: Some(json!({"topic": "custom_topic"})),
expected_status: 200,
check: |events| {
events.len() == 1
&& events[0].metadata.redirect_to_topic == Some("custom_topic".to_string())
},
},
];
for case in &cases {
let service = make_restriction_service(vec![Restriction {
restriction_type: case.restriction_type,
scope: RestrictionScope::AllEvents,
args: case.args.clone(),
}])
.await;
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
event_restriction_service: Some(service),
..Default::default()
},
);
let request = make_single_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, case.expected_status, "failed for: {}", case.name);
let events = sink.get_events().await;
assert!((case.check)(&events), "check failed for: {}", case.name);
}
}
#[tokio::test]
async fn test_filtered_drop_restriction_rejects_otel_batch() {
let mut event_names = HashSet::new();
event_names.insert("$ai_generation".to_string());
let service = make_restriction_service(vec![Restriction {
restriction_type: RestrictionType::DropEvent,
scope: RestrictionScope::Filtered(RestrictionFilters {
event_names,
..Default::default()
}),
args: None,
}])
.await;
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
event_restriction_service: Some(service),
..Default::default()
},
);
let request = make_two_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 400);
// The $ai_generation span matches the filtered drop restriction, so the
// entire batch is rejected (all-or-nothing semantics).
let events = sink.get_events().await;
assert!(events.is_empty());
}
// ============================================================================
// OverflowLimiter coverage for the OTEL endpoint
// ============================================================================
// `make_three_span_request` sets `posthog.distinct_id` only on the resource, so all its spans share one `token:distinct_id` key.
fn make_three_span_request() -> ExportTraceServiceRequest {
ExportTraceServiceRequest {
resource_spans: vec![ResourceSpans {
resource: Some(Resource {
attributes: vec![make_kv(
"posthog.distinct_id",
any_value::Value::StringValue("user-1".to_string()),
)],
dropped_attributes_count: 0,
}),
scope_spans: vec![ScopeSpans {
scope: None,
spans: vec![
make_span(
vec![1; 16],
vec![1; 8],
vec![],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
make_span(
vec![1; 16],
vec![2; 8],
vec![1; 8],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
make_span(
vec![1; 16],
vec![3; 8],
vec![1; 8],
1_704_067_200_000_000_000,
vec![make_kv(
"gen_ai.operation.name",
any_value::Value::StringValue("chat".to_string()),
)],
),
],
schema_url: String::new(),
}],
schema_url: String::new(),
}],
}
}
#[tokio::test]
async fn test_otel_batch_with_hot_token_stamps_force_limited_on_every_span() {
// Hot-list the request's `token:distinct_id`. Because OTEL derives one
// distinct_id per request (see identity::extract_distinct_id), every span
// in the batch shares the key, so every span is stamped ForceLimited.
let hot_key = format!("{TOKEN}:user-1");
let overflow_limiter = Arc::new(OverflowLimiter::new(
NonZeroU32::new(1_000).unwrap(),
NonZeroU32::new(1_000).unwrap(),
Some(hot_key),
true, // preserve_locality
));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_events_overflow_limiter: Some(overflow_limiter),
..Default::default()
},
);
let request = make_three_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 3);
for (i, event) in events.iter().enumerate() {
assert_eq!(
event.metadata.overflow_reason,
Some(OverflowReason::ForceLimited),
"span[{i}] must be stamped ForceLimited"
);
assert!(
event.metadata.skip_person_processing,
"span[{i}] ForceLimited implies skip_person_processing"
);
assert_eq!(event.metadata.data_type, DataType::AiEvents);
}
}
#[tokio::test]
async fn test_otel_batch_rate_limited_key_stamps_overbudget_spans() {
// burst=1, per_second=1: the first span in the batch fits, subsequent
// spans exhaust the budget and get RateLimited. This proves the helper
// runs per-event within the OTEL Vec (not once-per-request) and that the
// `preserve_locality` flag is mirrored faithfully onto the reason.
let overflow_limiter = Arc::new(OverflowLimiter::new(
NonZeroU32::new(1).unwrap(),
NonZeroU32::new(1).unwrap(),
None,
true, // preserve_locality
));
let sink = CapturingSink::new();
let client = make_test_client_with_options(
&sink,
TestClientOptions {
ai_events_overflow_limiter: Some(overflow_limiter),
..Default::default()
},
);
let request = make_three_span_request();
let status = send_request_with_client(&client, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 3);
assert_eq!(
events[0].metadata.overflow_reason, None,
"first span fits within the burst"
);
for (i, event) in events.iter().enumerate().skip(1) {
assert_eq!(
event.metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: true
}),
"span[{i}] must be stamped RateLimited{{preserve_locality: true}}"
);
assert!(
!event.metadata.skip_person_processing,
"span[{i}] RateLimited does NOT imply skip_person_processing"
);
}
}
#[tokio::test]
async fn test_otel_batch_without_overflow_limiter_leaves_reason_none() {
let sink = CapturingSink::new();
let request = make_three_span_request();
let status = send_request(&sink, &request).await;
assert_eq!(status, 200);
let events = sink.get_events().await;
assert_eq!(events.len(), 3);
for event in &events {
assert_eq!(event.metadata.overflow_reason, None);
assert!(!event.metadata.skip_person_processing);
}
}