:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use axum::http::HeaderMap;
use base64::Engine;
use chrono::{DateTime, TimeZone};
use common_types::RawEvent;
use rand::RngCore;
use std::collections::HashSet;
use tracing::debug;
use uuid::Uuid;
use crate::{
api::CaptureError,
payload::{Compression, EventFormData, EventQuery},
token::validate_token,
};
// used to limit test scans and extract loggable snippets from potentially large strings/buffers
pub const MAX_CHARS_TO_CHECK: usize = 128;
pub const MAX_PAYLOAD_SNIPPET_SIZE: usize = 20;
pub const FORM_MIME_TYPE: &str = "application/x-www-form-urlencoded";
#[derive(PartialEq, Eq)]
pub enum Base64Option {
// hasn't been decoded from urlencoded payload; won't include spaces
Strict,
// input might have been urlencoded; might include spaces that need touching up
Loose,
// data may have originated in a URL query param (GET request) and may use alt characters
URL,
}
pub fn random_bytes<const N: usize>() -> [u8; N] {
let mut ret = [0u8; N];
rand::thread_rng().fill_bytes(&mut ret);
ret
}
// basically just ripped from the uuid crate. they have it as unstable, but we can use it fine.
const fn encode_unix_timestamp_millis(millis: u64, random_bytes: &[u8; 10]) -> Uuid {
let millis_high = ((millis >> 16) & 0xFFFF_FFFF) as u32;
let millis_low = (millis & 0xFFFF) as u16;
let random_and_version =
(random_bytes[0] as u16 | ((random_bytes[1] as u16) << 8) & 0x0FFF) | (0x7 << 12);
let mut d4 = [0; 8];
d4[0] = (random_bytes[2] & 0x3F) | 0x80;
d4[1] = random_bytes[3];
d4[2] = random_bytes[4];
d4[3] = random_bytes[5];
d4[4] = random_bytes[6];
d4[5] = random_bytes[7];
d4[6] = random_bytes[8];
d4[7] = random_bytes[9];
Uuid::from_fields(millis_high, millis_low, random_and_version, &d4)
}
/// Builds a UUIDv7 whose time component is the given Unix-ms timestamp.
pub fn uuid_v7(unix_millis: u64) -> Uuid {
let bytes = random_bytes();
encode_unix_timestamp_millis(unix_millis, &bytes)
}
/// Builds a UUIDv7 whose time component is the given instant. Pre-epoch datetimes are floored to the epoch, since the unsigned time field can't represent negatives.
pub fn uuid_v7_from_datetime<Tz: TimeZone>(datetime: DateTime<Tz>) -> Uuid {
uuid_v7(datetime.timestamp_millis().max(0) as u64)
}
/// The high 48 bits of a UUIDv7 hold the Unix-millisecond timestamp.
pub fn uuid_time_millis(uuid: Uuid) -> u128 {
uuid.as_u128() >> 80
}
/// The instant a UUIDv7 encodes, or `None` for any other version. Capture mints a
/// UUIDv7 from the already-adjusted timestamp when the SDK sends none, so a caller
/// that compares this against the stored timestamp must use only client-sent UUIDs.
pub fn client_capture_millis(uuid: Uuid) -> Option<i64> {
if uuid.get_version_num() != 7 {
return None;
}
i64::try_from(uuid_time_millis(uuid)).ok()
}
// the compression hint can be tucked away any number of places depending on the SDK submitting the request...
pub fn extract_compression(
form: &EventFormData,
params: &EventQuery,
headers: &HeaderMap,
) -> Compression {
if params
.compression
.is_some_and(|c| c != Compression::Unsupported)
{
params.compression.unwrap()
} else if form
.compression
.is_some_and(|c| c != Compression::Unsupported)
{
form.compression.unwrap()
} else if let Some(ct) = headers.get("content-encoding") {
match ct.to_str().unwrap_or("UNKNOWN") {
"gzip" | "gzip-js" => Compression::Gzip,
"lz64" | "lz-string" => Compression::LZString,
"base64" | "b64" => Compression::Base64,
_ => Compression::Unsupported,
}
} else {
Compression::Unsupported
}
}
// have we decoded sufficiently have a urlencoded data payload of the expected form yet?
pub fn is_likely_urlencoded_form(payload: &[u8]) -> bool {
let prefix = &payload[..std::cmp::min(payload.len(), MAX_CHARS_TO_CHECK)];
[&b"data="[..], &b"_="[..], &b"ip="[..], &b"compression="[..]]
.iter()
.any(|target: &&[u8]| {
prefix.starts_with(target)
|| prefix
.windows(target.len() + 1)
.any(|window| window[0] == b'&' && &window[1..] == *target)
})
}
// relatively cheap check for base64 encoded payload since these can show up at
// various decoding layers in requests from different PostHog SDKs and versions
pub fn is_likely_base64(payload: &[u8], opt: Base64Option) -> bool {
if payload.is_empty() {
return false;
}
let prefix_chars_b64_compatible = payload.iter().take(MAX_CHARS_TO_CHECK).all(|b| {
(*b >= b'A' && *b <= b'Z')
|| (*b >= b'a' && *b <= b'z')
|| (*b >= b'0' && *b <= b'9')
|| (opt != Base64Option::URL && (*b == b'+' || *b == b'/' || *b == b'='))
|| (opt == Base64Option::URL && (*b == b'_' || *b == b'-'))
|| (opt == Base64Option::Loose && *b == b' ')
});
let is_b64_aligned = payload.len().is_multiple_of(4);
prefix_chars_b64_compatible && is_b64_aligned
}
pub fn decode_base64(payload: &[u8], location: &str) -> Result<Vec<u8>, CaptureError> {
// TODO(eli): parameterize to use general_purpose::URL_SAFE_NO_PAD engine for GET req payloads
match base64::engine::general_purpose::STANDARD.decode(payload) {
Ok(decoded_payload) => Ok(decoded_payload),
Err(e) => {
let max_chars = std::cmp::min(payload.len(), MAX_PAYLOAD_SNIPPET_SIZE);
let data_snippet = String::from_utf8(payload[..max_chars].to_vec())
.unwrap_or(String::from("INVALID_UTF8"));
debug!(
location = location,
data_snippet = data_snippet,
"decode_base64 failure: {}",
e
);
Err(CaptureError::RequestDecodingError(String::from(
"attempting to decode base64",
)))
}
}
}
pub fn decode_form(payload: &[u8]) -> Result<EventFormData, CaptureError> {
match serde_urlencoded::from_bytes::<EventFormData>(payload) {
Ok(form) => Ok(form),
Err(e) => {
let max_chars: usize = std::cmp::min(payload.len(), MAX_PAYLOAD_SNIPPET_SIZE);
let form_data_snippet = String::from_utf8(payload[..max_chars].to_vec())
.unwrap_or(String::from("INVALID_UTF8"));
debug!(
form_data = form_data_snippet,
"failed to decode urlencoded form body: {e:#}"
);
Err(CaptureError::RequestDecodingError(String::from(
"invalid urlencoded form data",
)))
}
}
}
pub fn extract_and_verify_token(
events: &[RawEvent],
batch_token: Option<String>,
) -> Result<String, CaptureError> {
// if the request was a batch submission and it's token is valid, give it precedence
let token = if let Some(token) = batch_token {
token
} else {
match events.len() {
0 => return Err(CaptureError::NoTokenError),
1 => events[0]
.extract_token()
.ok_or(CaptureError::NoTokenError)?,
_ => extract_token(events)?,
}
};
validate_token(&token)?;
Ok(token)
}
pub fn extract_token(events: &[RawEvent]) -> Result<String, CaptureError> {
let distinct_tokens: HashSet<Option<String>> = HashSet::from_iter(
events
.iter()
.map(RawEvent::extract_token)
.filter(Option::is_some),
);
match distinct_tokens.len() {
0 => Err(CaptureError::NoTokenError),
1 => match distinct_tokens.iter().last() {
Some(Some(token)) => Ok(token.clone()),
_ => Err(CaptureError::NoTokenError),
},
_ => Err(CaptureError::MultipleTokensError),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uuid_v7_from_datetime_encodes_the_instant() {
// A non-UTC offset still yields the same absolute instant in the UUID.
let ts = DateTime::parse_from_rfc3339("2020-06-15T00:00:00Z").unwrap();
let uuid = uuid_v7_from_datetime(ts);
assert_eq!(uuid_time_millis(uuid), ts.timestamp_millis() as u128);
assert_eq!(uuid.get_version_num(), 7);
}
#[test]
fn uuid_v7_from_datetime_floors_pre_epoch_to_zero() {
let ts = DateTime::parse_from_rfc3339("1969-06-15T00:00:00Z").unwrap();
assert_eq!(uuid_time_millis(uuid_v7_from_datetime(ts)), 0);
}
}