:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use bytes::Bytes;
use chrono::{DateTime, Utc};
use common_types::{CapturedEvent, RawEngageEvent, RawEvent};
use serde::Deserialize;
use time::format_description::well_known::Iso8601;
use time::OffsetDateTime;
use tracing::{instrument, Span};
use crate::{
api::CaptureError,
config::CaptureMode,
event_restrictions::Pipeline,
ingestion_warnings::SdkAttribution,
payload::{decompress_payload, Compression},
};
#[derive(Deserialize)]
#[serde(untagged)]
pub enum RawRequest {
/// Array of events (posthog-js)
Array(Vec<RawEvent>),
/// Batched events (/batch)
Batch(BatchedRequest),
/// Single event (/capture)
One(Box<RawEvent>),
/// Single person-props update event w/o name (/engage)
Engage(Box<RawEngageEvent>),
}
#[derive(Deserialize)]
pub struct BatchedRequest {
#[serde(alias = "api_key")]
pub token: String,
pub historical_migration: Option<bool>,
pub sent_at: Option<String>,
pub batch: Vec<RawEvent>,
}
impl RawRequest {
/// Takes a request payload and tries to decompress and unmarshall it.
/// While posthog-js sends a compression query param, a sizable portion of requests
/// fail due to it being missing when the body is compressed.
/// Instead of trusting the parameter, we peek at the payload's first three bytes to
/// detect gzip, fallback to uncompressed utf8 otherwise.
#[instrument(skip_all, fields(request_id, compression, is_mirror_deploy))]
pub fn from_bytes(
bytes: Bytes,
cmp_hint: Compression,
request_id: &str,
limit: usize,
path: String,
) -> Result<RawRequest, CaptureError> {
Span::current().record("compression", cmp_hint.to_string());
Span::current().record("path", path.clone());
Span::current().record("request_id", request_id);
// Use shared decompression logic
let payload = decompress_payload(bytes, cmp_hint, limit, &path)?;
Ok(serde_json::from_str::<RawRequest>(&payload)?)
}
pub fn get_batch_token(&self) -> Option<String> {
if let RawRequest::Batch(req) = self {
return Some(req.token.clone());
}
None
}
pub fn events(self, path: &str) -> Result<Vec<RawEvent>, CaptureError> {
let result = match self {
RawRequest::Array(events) => Ok(events),
RawRequest::One(event) => Ok(vec![*event]),
RawRequest::Batch(req) => Ok(req.batch),
RawRequest::Engage(engage_event) => {
if path.starts_with("/engage") {
Ok(vec![RawEvent {
event: String::from("$identify"),
token: engage_event.token,
distinct_id: engage_event.distinct_id,
uuid: engage_event.uuid,
timestamp: engage_event.timestamp,
offset: engage_event.offset,
set: engage_event.set,
set_once: engage_event.set_once,
properties: engage_event.properties,
}])
} else {
Err(CaptureError::RequestHydrationError(String::from(
"non-engage request missing event name attribute",
)))
}
}
};
// do some basic hydrated event payload filtering here
match result {
Ok(mut events) => {
if events.is_empty() {
return Err(CaptureError::EmptyBatch);
}
// filter event types we don't want to ingest; return a sentinel
// error response if this results in an empty payload
events.retain(|event| event.event != "$performance_event");
if events.is_empty() {
return Err(CaptureError::EmptyPayloadFiltered);
}
Ok(events)
}
// pass along payload hydration and other error types
_ => result,
}
}
pub fn historical_migration(&self) -> bool {
match self {
RawRequest::Batch(req) => req.historical_migration.unwrap_or_default(),
_ => false,
}
}
pub fn sent_at(&self) -> Option<OffsetDateTime> {
if let RawRequest::Batch(req) = &self {
if let Some(value) = &req.sent_at {
if let Ok(parsed) = OffsetDateTime::parse(value, &Iso8601::DEFAULT) {
return Some(parsed);
}
}
}
None
}
}
#[derive(Debug)]
pub struct ProcessingContext {
pub user_agent: Option<String>,
pub sent_at: Option<OffsetDateTime>,
pub token: String,
pub now: DateTime<Utc>,
pub client_ip: String,
pub request_id: String,
pub path: String,
pub is_mirror_deploy: bool, // TODO(eli): can remove after migration
pub historical_migration: bool,
pub chatty_debug_enabled: bool,
/// Deployment capture mode. Governs Import-only policy in the legacy
/// analytics path: skip the global rate limiter and drop any batch not
/// flagged `historical_migration: true`.
pub capture_mode: CaptureMode,
/// Largest AI-lane event this deployment accepts, from
/// `AI_MAX_EVENT_BYTES`. `0` disables the ceiling. See
/// [`exceeds_max_ai_event_bytes`].
pub ai_max_event_bytes: u64,
/// SDK identity snapshotted from the batch's first event, for ingestion
/// warning attribution. Captured at batch construction because the events
/// are typed there; downstream stages hold serialized payloads and would
/// have to re-parse JSON on the hot path to recover it. Display only —
/// nothing routes on it. See [`crate::ingestion_warnings`].
pub sdk_attribution: SdkAttribution,
}
// these are the legacy endpoints capture maintains. Can eliminate this
// during post-migration refactoring, once we validate we can safely unite
// the legacy and "common" handling flows
pub fn path_is_legacy_endpoint(path: &str) -> bool {
path == "/e"
|| path.starts_with("/e/")
|| path.starts_with("/e?")
|| path.starts_with("/capture")
|| path.starts_with("/engage")
|| path.starts_with("/track")
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[cfg_attr(test, derive(strum::EnumIter))]
pub enum DataType {
AnalyticsMain,
AnalyticsHistorical,
ClientIngestionWarning,
HeatmapMain,
ExceptionErrorTracking,
SnapshotMain,
/// Dedicated AI lane, mirroring v1's `Destination::AiMain`. The
/// kafka sink maps it to `CAPTURE_OUTPUT_AI_MAIN_TOPIC`, so every
/// deployment that accepts AI traffic must configure that topic. Like
/// heatmaps and exceptions, AI events never reroute historical.
AiEvents,
}
pub const AI_LANE_NAME_PREFIX: &str = "$ai_";
/// The single AI-lane membership rule; the ingestion AI pipeline admits by the
/// same prefix (`isAiEventName` in `nodejs/src/ingestion/common/ai-event-types.ts`).
pub fn is_ai_event(name: &str) -> bool {
name.starts_with(AI_LANE_NAME_PREFIX)
}
/// Whether an AI-lane event's body exceeds the deployment's per-event ceiling.
///
/// The ceiling exists because each deployment's AI topic sits on a broker with
/// its own message limit, and the producer's own cap rejects an oversized event
/// only after capture has read and processed the whole request. `0` disables
/// the check.
///
/// `body_bytes` is whatever the calling path can measure cheaply, and the
/// paths differ: v0, OTEL, and the multipart handler pass the serialized
/// event, while v1 passes the properties blob, which dominates an AI event
/// but omits the event name, distinct_id, uuid, timestamp, and their keys.
/// The ceiling is sized with broker headroom well above that difference, so
/// the looser measure still keeps an accepted event under the producer's cap.
pub fn exceeds_max_ai_event_bytes(body_bytes: usize, max: u64) -> bool {
max != 0 && body_bytes as u64 > max
}
impl DataType {
/// Classify an event by its name (and historical-migration flag) into a
/// `DataType`. Used by both v0's `process_single_event` and v1's
/// `apply_restrictions` so the analytics → exception → heatmap →
/// ingestion-warning split stays in one place.
///
/// AI events (per [`is_ai_event`]) divert to `AiEvents`
/// on every deployment, winning over historical (in v1 the historical
/// reroute only applies to the analytics-main destination). Mirrors v1's
/// `destination_for_event_name`.
///
/// `SnapshotMain` is not produced here — replay events arrive on a
/// separate endpoint and never flow through analytics processing.
pub fn from_event_name(event_name: &str, historical_migration: bool) -> Self {
match event_name {
"$$client_ingestion_warning" => Self::ClientIngestionWarning,
"$exception" => Self::ExceptionErrorTracking,
"$$heatmap" => Self::HeatmapMain,
_ if is_ai_event(event_name) => Self::AiEvents,
_ if historical_migration => Self::AnalyticsHistorical,
_ => Self::AnalyticsMain,
}
}
/// Restriction pipeline this event is looked up under, if any. Snapshots
/// have none here: the recordings endpoint looks them up itself.
///
/// `AiEvents` maps to the `ai` restriction pipeline: an event on the AI
/// lane is governed by ai-scoped restrictions, the same slice the
/// dedicated AI endpoints consult. v1 derives the same mapping from
/// `Destination::pipeline`.
pub fn pipeline(self) -> Option<Pipeline> {
match self {
Self::AnalyticsMain | Self::AnalyticsHistorical => Some(Pipeline::Analytics),
Self::AiEvents => Some(Pipeline::Ai),
Self::ExceptionErrorTracking => Some(Pipeline::ErrorTracking),
Self::HeatmapMain => Some(Pipeline::Heatmaps),
Self::ClientIngestionWarning => Some(Pipeline::ClientWarnings),
Self::SnapshotMain => None,
}
}
}
#[derive(Debug, Clone)]
pub struct ProcessedEvent {
pub metadata: ProcessedEventMetadata,
pub event: CapturedEvent,
}
/// Reason the pipeline has decided to reroute an event to an overflow topic.
///
/// Set by every handler that emits events to the kafka sink, via the shared
/// `events::overflow_stamping::stamp_overflow_reason` helper for the
/// in-process `OverflowLimiter` (governor-backed) paths and a separate inline
/// check for the replay `RedisLimiter` (session-scoped, redis-backed). Call
/// sites:
/// * `events::analytics::process_events` — `/e/`, `/batch/`, `/capture`, etc.
/// * `events::recordings::process_replay_events` — `/s/` (stamps `ReplayLimited`)
/// * `ai_endpoint::ai_handler` — `/i/v0/ai`
/// * `otel::otel_handler` — `/i/v0/ai/otel`
///
/// Consumed by `sinks::kafka::KafkaSinkBase::prepare_record`, which is pure
/// mechanism: it reads this reason and maps to the overflow topic and
/// partition key. The sink does not make routing policy decisions of its own
/// for overflow beyond this mapping.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OverflowReason {
/// The overflow governor matched a configured `keys_to_reroute` entry
/// for this event's key, or the global rate limiter put the key over its
/// window. Routed to the overflow topic with a null partition key, with
/// person processing off: the reason itself implies the skip
/// ([`ProcessedEventMetadata::person_processing_disabled`]), so a
/// stamping site cannot keep person processing on for a force-limited
/// key by forgetting the flag. Stamping sites still set
/// `skip_person_processing = true` alongside for pipeline-level readers.
ForceLimited,
/// Per-key rate exceeded the configured governor quota. Routed to the
/// overflow topic. `preserve_locality` mirrors the
/// `overflow_preserve_partition_locality` config and determines whether
/// the original partition key is preserved on the overflow topic.
RateLimited { preserve_locality: bool },
/// Session-level replay overflow signalled by the redis-backed limiter.
/// Routed to the replay overflow topic, keyed on session_id.
ReplayLimited,
}
#[derive(Debug, Clone)]
pub struct ProcessedEventMetadata {
pub data_type: DataType,
pub session_id: Option<String>,
pub computed_timestamp: Option<chrono::DateTime<chrono::Utc>>,
pub event_name: String,
/// Force this event to overflow topic (set by event restrictions)
pub force_overflow: bool,
/// Skip person processing for this event (set by event restrictions)
pub skip_person_processing: bool,
/// Redirect this event to DLQ topic (set by event restrictions)
pub redirect_to_dlq: bool,
/// Redirect this event to a custom topic (set by event restrictions)
pub redirect_to_topic: Option<String>,
/// Heatmap data on this event was redirected to the heatmaps topic and
/// must not be extracted again by the events pipeline.
pub skip_heatmap_processing: bool,
/// Overflow routing decision stamped by the pipeline. `None` means the
/// event stays on its default topic for its `data_type`. See
/// [`OverflowReason`] for who sets this and what each variant maps to in
/// the kafka sink.
pub overflow_reason: Option<OverflowReason>,
/// Char count of the event's original distinct_id when extraction cut it
/// down to the 200-char cap; `None` when it was ingested unmodified.
/// Feeds the `distinct_id_truncated` ingestion warning; nothing routes on
/// it.
pub distinct_id_truncated_from: Option<usize>,
}
impl ProcessedEventMetadata {
/// Whether person processing is off for this event: the stamped flag, or
/// a `ForceLimited` reason — force-limiting implies the skip, so the
/// sink's header and ordering cannot silently keep person processing on
/// for a force-limited key whose stamping site forgot the flag.
pub fn person_processing_disabled(&self) -> bool {
self.skip_person_processing
|| matches!(self.overflow_reason, Some(OverflowReason::ForceLimited))
}
}
#[cfg(test)]
mod tests {
use crate::token::InvalidTokenReason;
use crate::utils::extract_and_verify_token;
use base64::Engine as _;
use bytes::Bytes;
use common_types::RawEvent;
use rand::distributions::Alphanumeric;
use rand::Rng;
use serde::Deserialize;
use serde_json::json;
use super::{CaptureError, Compression, DataType, RawRequest};
/// Mirrors v1's `destination_for_event_name` mapping tests: the
/// dedicated-name lanes always win, an AI name diverts on every deployment
/// (beating historical), and everything else falls through to
/// main/historical per the batch flag.
#[rstest::rstest]
// Dedicated-name lanes are unaffected by the historical flag.
#[case("$exception", false, DataType::ExceptionErrorTracking)]
#[case("$exception", true, DataType::ExceptionErrorTracking)]
#[case("$$heatmap", false, DataType::HeatmapMain)]
#[case("$$heatmap", true, DataType::HeatmapMain)]
#[case("$$client_ingestion_warning", false, DataType::ClientIngestionWarning)]
#[case("$$client_ingestion_warning", true, DataType::ClientIngestionWarning)]
// Non-AI events keep the main/historical split.
#[case("$pageview", false, DataType::AnalyticsMain)]
#[case("custom_event", false, DataType::AnalyticsMain)]
#[case("$pageview", true, DataType::AnalyticsHistorical)]
// Names that only look like the prefix never divert.
#[case("ai_generation", false, DataType::AnalyticsMain)]
#[case("$AI_generation", false, DataType::AnalyticsMain)]
#[case("$ai", false, DataType::AnalyticsMain)]
// Every `$ai_` name diverts, and wins over historical.
#[case("$ai_generation", false, DataType::AiEvents)]
#[case("$ai_span", false, DataType::AiEvents)]
#[case("$ai_trace", false, DataType::AiEvents)]
#[case("$ai_generation_summary", false, DataType::AiEvents)]
#[case("$ai_generation", true, DataType::AiEvents)]
#[case("$ai_call", false, DataType::AiEvents)]
#[case("$ai_generation_enriched", false, DataType::AiEvents)]
#[case("$ai_model_failover", false, DataType::AiEvents)]
#[case("$ai_model_failover", true, DataType::AiEvents)]
#[case("$ai_", false, DataType::AiEvents)]
fn from_event_name_mapping(
#[case] event_name: &str,
#[case] historical_migration: bool,
#[case] expected: DataType,
) {
assert_eq!(
DataType::from_event_name(event_name, historical_migration),
expected
);
}
#[test]
fn decode_compression_param() {
#[derive(Deserialize, Debug)]
struct TestConfig {
compression: Option<Compression>,
}
struct CompressionUnit {
input: &'static str,
output: Option<Compression>,
}
let units = vec![
CompressionUnit {
input: r#"{"compression": "gzip"}"#,
output: Some(Compression::Gzip),
},
CompressionUnit {
input: r#"{"compression": "gzip-js"}"#,
output: Some(Compression::Gzip),
},
CompressionUnit {
input: r#"{"compression": "GZIP"}"#,
output: Some(Compression::Gzip),
},
CompressionUnit {
input: r#"{"compression": "lz64"}"#,
output: Some(Compression::LZString),
},
CompressionUnit {
input: r#"{"compression": "lz-string"}"#,
output: Some(Compression::LZString),
},
CompressionUnit {
input: r#"{"compression": "LZ64"}"#,
output: Some(Compression::LZString),
},
CompressionUnit {
input: r#"{"compression": "base64"}"#,
output: Some(Compression::Base64),
},
CompressionUnit {
input: r#"{"compression": "b64"}"#,
output: Some(Compression::Base64),
},
CompressionUnit {
input: r#"{"compression": "BASE64"}"#,
output: Some(Compression::Base64),
},
CompressionUnit {
input: r#"{"compression": "foobar"}"#,
output: Some(Compression::Unsupported),
},
CompressionUnit {
input: r#"{"compression": ""}"#,
output: Some(Compression::Unsupported),
},
CompressionUnit {
input: "{}", // no compression param set
output: None,
},
];
for unit in units {
let result: Result<TestConfig, _> = serde_json::from_str(unit.input);
assert!(
result.is_ok(),
"result was not OK for input({}): {:?}",
unit.input,
result
);
let got = result.unwrap().compression;
assert!(
got == unit.output,
"result {:?} didn't match expected {:?}",
got,
unit.output
);
}
}
#[test]
fn decode_uncompressed_raw_event() {
let base64_payload = "ewogICAgImRpc3RpbmN0X2lkIjogIm15X2lkMSIsCiAgICAiZXZlbnQiOiAibXlfZXZlbnQxIiwKICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICIkZGV2aWNlX3R5cGUiOiAiRGVza3RvcCIKICAgIH0sCiAgICAiYXBpX2tleSI6ICJteV90b2tlbjEiCn0K";
let compressed_bytes = Bytes::from(
base64::engine::general_purpose::STANDARD
.decode(base64_payload)
.expect("payload is not base64"),
);
let path = "/i/v0/e";
let events = RawRequest::from_bytes(
compressed_bytes,
Compression::Unsupported,
"decode_uncompressed_raw_event",
1024,
path.to_string(),
)
.expect("failed to parse")
.events(path)
.unwrap();
assert_eq!(1, events.len());
assert_eq!(Some("my_token1".to_string()), events[0].extract_token());
assert_eq!("my_event1".to_string(), events[0].event);
assert_eq!(
"my_id1".to_string(),
events[0]
.extract_distinct_id()
.expect("cannot find distinct_id")
);
}
#[test]
fn decode_gzipped_raw_event() {
let base64_payload = "H4sIADQSbmUCAz2MsQqAMAxE936FBEcnR2f/o4i9IRTb0AahiP9urcVMx3t3ucxQjxxn5bCrZUfLQEepYabpkzgRtOOWfyMpCpIyctVXY42PDifvsFoE73BF9hqFWuPu403YepT+WKNHmMnc5gENoFu2kwAAAA==";
let compressed_bytes = Bytes::from(
base64::engine::general_purpose::STANDARD
.decode(base64_payload)
.expect("payload is not base64"),
);
let path = "/i/v0/e";
let events = RawRequest::from_bytes(
compressed_bytes,
Compression::Unsupported,
"decode_gzipped_raw_event",
2048,
path.to_string(),
)
.expect("failed to parse")
.events(path)
.unwrap();
assert_eq!(1, events.len());
assert_eq!(Some("my_token2".to_string()), events[0].extract_token());
assert_eq!("my_event2".to_string(), events[0].event);
assert_eq!(
"my_id2".to_string(),
events[0]
.extract_distinct_id()
.expect("cannot find distinct_id")
);
}
#[test]
fn extract_non_engage_event_without_name_fails() {
let path = "/e/?ip=192.0.0.1";
let parse_and_extract_events =
|input: &'static str| -> Result<Vec<RawEvent>, CaptureError> {
RawRequest::from_bytes(
input.into(),
Compression::Unsupported,
"extract_distinct_id",
2048,
path.to_string(),
)
.expect("failed to parse")
.events(path)
};
// since we're not extracting events against the /engage endpoint path,
// an event with a missing "event" (name) attribute is invalid
assert!(matches!(
parse_and_extract_events(
r#"{"token": "token", "distinct_id": "distinct_id", "properties":{"foo": 42, "bar": true}}"#
),
Err(CaptureError::RequestHydrationError(_))
));
}
#[test]
fn extract_engage_event_without_name_is_resolved() {
let path = "/engage/?ip=10.0.0.1";
let parse_and_extract_events =
|input: &'static str| -> Result<Vec<RawEvent>, CaptureError> {
RawRequest::from_bytes(
input.into(),
Compression::Unsupported,
"extract_distinct_id",
2048,
path.to_string(),
)
.expect("failed to parse")
.events(path)
};
let got = parse_and_extract_events(
r#"{"token": "token", "distinct_id": "distinct_id", "$set":{"foo": 42, "bar": true}}"#,
)
.expect("engage event hydrated");
assert!(got.len() == 1);
assert!(&got[0].event == "$identify");
}
#[test]
fn extract_distinct_id() {
let path = "/i/v0/e";
let parse_and_extract = |input: &'static str| -> Result<String, CaptureError> {
let parsed = RawRequest::from_bytes(
input.into(),
Compression::Unsupported,
"extract_distinct_id",
2048,
path.to_string(),
)
.expect("failed to parse")
.events(path)
.unwrap();
parsed[0]
.extract_distinct_id()
.ok_or(CaptureError::MissingDistinctId)
};
// Return MissingDistinctId if not found
assert!(matches!(
parse_and_extract(r#"{"event": "e"}"#),
Err(CaptureError::MissingDistinctId)
));
// Return MissingDistinctId if null
assert!(matches!(
parse_and_extract(r#"{"event": "e", "distinct_id": null}"#),
Err(CaptureError::MissingDistinctId)
));
// Return EmptyDistinctId if empty string
assert!(matches!(
parse_and_extract(r#"{"event": "e", "distinct_id": ""}"#),
Err(CaptureError::MissingDistinctId)
));
let assert_extracted_id = |input: &'static str, expected: &str| {
let id = parse_and_extract(input).expect("failed to extract");
assert_eq!(id, expected);
};
// Happy path: toplevel field present
assert_extracted_id(r#"{"event": "e", "distinct_id": "myid"}"#, "myid");
assert_extracted_id(r#"{"event": "e", "$distinct_id": "23"}"#, "23");
// Sourced from properties if not present in toplevel field, but toplevel wins if both present
assert_extracted_id(
r#"{"event": "e", "properties":{"distinct_id": "myid"}}"#,
"myid",
);
assert_extracted_id(
r#"{"event": "e", "distinct_id": 23, "properties":{"distinct_id": "myid"}}"#,
"23",
);
// Numbers are stringified
assert_extracted_id(r#"{"event": "e", "distinct_id": 23}"#, "23");
assert_extracted_id(r#"{"event": "e", "distinct_id": 23.4}"#, "23.4");
// Containers are stringified
assert_extracted_id(
r#"{"event": "e", "distinct_id": ["a", "b"]}"#,
r#"["a","b"]"#,
);
assert_extracted_id(
r#"{"event": "e", "distinct_id": {"string": "a", "number": 3}}"#,
r#"{"number":3,"string":"a"}"#,
);
}
#[test]
fn extract_distinct_id_trims_to_200_chars() {
let distinct_id: String = rand::thread_rng()
.sample_iter(Alphanumeric)
.take(222)
.map(char::from)
.collect();
let (expected_distinct_id, _) = distinct_id.split_at(200); // works because ascii chars only
let input = json!([{
"token": "mytoken",
"event": "myevent",
"distinct_id": distinct_id
}]);
let path = "/i/v0/e";
let parsed = RawRequest::from_bytes(
input.to_string().into(),
Compression::Unsupported,
"extract_distinct_id_trims_to_200_chars",
2048,
path.to_string(),
)
.expect("failed to parse")
.events(path)
.unwrap();
assert_eq!(
parsed[0].extract_distinct_id().expect("failed to extract"),
expected_distinct_id
);
}
#[test]
fn test_gzip_bomb_protection() {
use flate2::write::GzEncoder;
use flate2::Compression as GzCompression;
use std::io::Write;
// Create a highly compressible payload (GZIP bomb)
// 10MB of zeros compresses to just a few KB
let uncompressed_size = 10 * 1024 * 1024; // 10MB
let zeros = vec![0u8; uncompressed_size];
// Wrap in JSON structure
let json_payload = format!(
r#"[{{"event":"test","distinct_id":"test","properties":{{"data":"{}"}}}}"#,
base64::engine::general_purpose::STANDARD.encode(&zeros)
);
// Compress with maximum compression
let mut encoder = GzEncoder::new(Vec::new(), GzCompression::best());
encoder
.write_all(json_payload.as_bytes())
.expect("Failed to write");
let compressed = encoder.finish().expect("Failed to compress");
let compressed_size = compressed.len();
let compression_ratio = uncompressed_size as f64 / compressed_size as f64;
// Verify we created a highly compressed payload
assert!(
compression_ratio > 100.0,
"Expected compression ratio > 100, got {compression_ratio}"
);
// Set a reasonable limit that should catch the bomb
let limit = 1024 * 1024; // 1MB limit
let path = "/i/v0/e";
let result = RawRequest::from_bytes(
Bytes::from(compressed),
Compression::Gzip,
"test_gzip_bomb",
limit,
path.to_string(),
);
// Should fail due to size limit
match result {
Err(CaptureError::EventTooBig(msg)) => {
assert!(
msg.contains("exceed"),
"Expected error message about exceeding limit, got: {msg}"
);
}
Ok(_) => panic!("GZIP bomb should have been rejected"),
Err(e) => panic!("Wrong error type: {e:?}"),
}
}
#[test]
fn test_gzip_normal_compression_allowed() {
use flate2::write::GzEncoder;
use flate2::Compression as GzCompression;
use std::io::Write;
// Create a normal JSON payload with realistic compression ratio
let json_payload = r#"[{
"event": "pageview",
"distinct_id": "user123",
"properties": {
"url": "https://example.com/page",
"referrer": "https://google.com",
"timestamp": "2024-01-01T00:00:00Z",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
}
}]"#;
// Compress with normal compression
let mut encoder = GzEncoder::new(Vec::new(), GzCompression::default());
encoder
.write_all(json_payload.as_bytes())
.expect("Failed to write");
let compressed = encoder.finish().expect("Failed to compress");
let compressed_size = compressed.len();
let uncompressed_size = json_payload.len();
let compression_ratio = uncompressed_size as f64 / compressed_size as f64;
// Normal JSON typically compresses 2-4x
assert!(
compression_ratio < 10.0,
"Expected normal compression ratio < 10, got {compression_ratio}"
);
// Should succeed with reasonable limit
let limit = 10 * 1024; // 10KB limit
let path = "/i/v0/e";
let result = RawRequest::from_bytes(
Bytes::from(compressed),
Compression::Gzip,
"test_normal_compression",
limit,
path.to_string(),
);
// Should succeed
match result {
Ok(req) => {
let events = req.events(path).expect("Failed to extract events");
assert_eq!(events.len(), 1);
assert_eq!(events[0].event, "pageview");
assert_eq!(events[0].extract_distinct_id(), Some("user123".to_string()));
}
Err(e) => panic!("Normal compressed payload should succeed: {e:?}"),
}
}
#[test]
fn test_gzip_decompression_size_check_happens_before_allocation() {
use flate2::write::GzEncoder;
use flate2::Compression as GzCompression;
use std::io::Write;
// Create a payload that's just under the limit when compressed
// but would exceed it when decompressed
let limit = 1024; // 1KB limit
// Create 2KB of JSON data (exceeds limit when decompressed)
let large_string = "x".repeat(2048);
let json_payload = format!(
r#"[{{"event":"test","distinct_id":"test","properties":{{"data":"{large_string}"}}}}"#
);
// Compress it (will be smaller than limit)
let mut encoder = GzEncoder::new(Vec::new(), GzCompression::best());
encoder
.write_all(json_payload.as_bytes())
.expect("Failed to write");
let compressed = encoder.finish().expect("Failed to compress");
assert!(
compressed.len() < limit,
"Compressed size {} should be less than limit {}",
compressed.len(),
limit
);
assert!(
json_payload.len() > limit,
"Uncompressed size {} should exceed limit {}",
json_payload.len(),
limit
);
let path = "/i/v0/e";
let result = RawRequest::from_bytes(
Bytes::from(compressed),
Compression::Gzip,
"test_size_check_before_alloc",
limit,
path.to_string(),
);
// Should fail due to decompressed size exceeding limit
match result {
Err(CaptureError::EventTooBig(msg)) => {
// Verify the error message indicates it caught the size during decompression
assert!(
msg.contains("would exceed") || msg.contains("exceed"),
"Expected error about exceeding size during decompression, got: {msg}"
);
}
Ok(_) => panic!("Should have rejected payload that exceeds limit when decompressed"),
Err(e) => panic!("Wrong error type: {e:?}"),
}
}
#[test]
fn test_extract_and_verify_token() {
let parse_and_extract = |input: &'static str| -> Result<String, CaptureError> {
let path = "/i/v0/e";
let raw_req = RawRequest::from_bytes(
input.into(),
Compression::Unsupported,
"extract_and_verify_token",
2048,
path.to_string(),
)
.expect("failed to parse");
let maybe_batch_token = raw_req.get_batch_token();
let events = raw_req
.events(path)
.expect("failed to hydrate Vec<RawEvent>");
extract_and_verify_token(&events, maybe_batch_token)
};
let assert_extracted_token = |input: &'static str, expected: &str| {
let id = parse_and_extract(input).expect("failed to extract");
assert_eq!(id, expected);
};
// Return NoTokenError if not found
assert!(matches!(
parse_and_extract(r#"{"event": "e"}"#),
Err(CaptureError::NoTokenError)
));
// Return TokenValidationError if token empty
assert!(matches!(
parse_and_extract(r#"{"api_key": "", "batch":[{"event": "e"}]}"#),
Err(CaptureError::TokenValidationError(
InvalidTokenReason::Empty
))
));
// Return TokenValidationError if personal apikey
assert!(matches!(
parse_and_extract(r#"[{"event": "e", "token": "phx_hellothere"}]"#),
Err(CaptureError::TokenValidationError(
InvalidTokenReason::PersonalApiKey
))
));
// Return MultipleTokensError if tokens don't match in array
assert!(matches!(
parse_and_extract(
r#"[{"event": "e", "token": "token1"},{"event": "e", "token": "token2"}]"#
),
Err(CaptureError::MultipleTokensError)
));
// Return token from array if consistent
assert_extracted_token(
r#"[{"event":"e","token":"token1"},{"event":"e","token":"token1"}]"#,
"token1",
);
// Return token from batch if present
assert_extracted_token(
r#"{"batch":[{"event":"e","token":"token1"}],"api_key":"batched"}"#,
"batched",
);
// Return token from single event if present
assert_extracted_token(r#"{"event":"e","$token":"single_token"}"#, "single_token");
assert_extracted_token(r#"{"event":"e","api_key":"single_token"}"#, "single_token");
}
}