:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::{
api::{
errors::FlagError,
types::{Compression, FlagsQueryParams},
},
flags::flag_request::FlagRequest,
metrics::consts::{FLAG_GZIP_OUTPUT_EXCEEDED_COUNTER, FLAG_REQUEST_KLUDGE_COUNTER},
utils::user_agent::UserAgentInfo,
};
use axum::http::{header::CONTENT_TYPE, header::USER_AGENT, HeaderMap};
use base64::{engine::general_purpose, Engine as _};
use bytes::Bytes;
use common_compression::{self, has_gzip_magic_header};
use common_metrics::inc;
use percent_encoding::percent_decode;
/// 4 MiB cap on the decompressed `/flags` request body. The compressed body is
/// already capped at 2 MiB by axum's `DefaultBodyLimit` (`MAX_FLAGS_BODY_BYTES`),
/// so this is a backstop against gzip-bomb amplification.
const MAX_FLAGS_DECOMPRESSED_BYTES: usize = 4 * 1024 * 1024;
/// Lightweight token extraction for rate limiting.
/// Tries to extract the token without full request deserialization.
/// Falls back to None if extraction fails (caller should use IP).
pub fn extract_token(body: &Bytes) -> Option<String> {
// Try to parse just the token field from JSON body
// This is much faster than full decode_request()
serde_json::from_slice::<serde_json::Value>(body)
.ok()
.and_then(|v| {
v.get("token")
.or_else(|| v.get("api_key"))
.or_else(|| v.get("$token"))
.and_then(|t| t.as_str())
.filter(|s| !s.is_empty())
.map(String::from)
})
}
/// Decode and parse a /flags request body.
///
/// Returns the parsed [`FlagRequest`] along with the **decoded** body bytes
/// — post gzip-decompression and post base64-decoding when those apply, or
/// the raw body for plain JSON. Callers that want to log or otherwise reuse
/// the parsed-form bytes (e.g. body logging) can avoid a second decompress
/// pass by reusing the returned `Bytes`.
pub fn decode_request(
headers: &HeaderMap,
body: Bytes,
query: &FlagsQueryParams,
) -> Result<(FlagRequest, Bytes), FlagError> {
let content_type = headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("application/json"); // Default to JSON if no content type
let base_content_type = content_type.split(';').next().unwrap_or("").trim();
let user_agent = headers.get(USER_AGENT).and_then(|v| v.to_str().ok());
match base_content_type {
"application/json" | "text/plain" => {
let decoded_body = decode_body(body, query.compression, headers)?;
try_parse_with_fallbacks(decoded_body, user_agent)
}
"application/x-www-form-urlencoded" => {
decode_form_data(body, query.compression, user_agent)
}
_ => {
tracing::warn!("unsupported content type: {}", content_type);
Err(FlagError::RequestDecodingError(format!(
"unsupported content type: {content_type}"
)))
}
}
}
/// Decode a request body to its raw JSON bytes.
///
/// Handles gzip explicitly via `Compression::Gzip`, the `Content-Encoding`
/// header, or auto-detection from magic bytes. `Compression::Base64` is left
/// to `try_parse_with_fallbacks`; this function returns the body unchanged in
/// that case.
pub(crate) fn decode_body(
body: Bytes,
compression: Option<Compression>,
headers: &HeaderMap,
) -> Result<Bytes, FlagError> {
if let Some(compression) = compression {
match compression {
Compression::Gzip => return decompress_gzip(body),
Compression::Base64 => {
// handle base64 detection separately in try_parse_with_fallbacks
}
Compression::Unsupported => {
tracing::warn!("unsupported compression type");
return Err(FlagError::RequestDecodingError(
"Unsupported compression type".to_string(),
));
}
}
}
// Check Content-Encoding header (Android uses this primarily)
if let Some(encoding) = headers.get("content-encoding") {
if let Ok(encoding_str) = encoding.to_str() {
if encoding_str.contains("gzip") {
tracing::debug!(
"Detected gzip from Content-Encoding header: {}",
encoding_str
);
return decompress_gzip(body);
}
}
}
// Fallback: Auto-detect gzip by checking magic bytes (0x1f, 0x8b)
// This handles cases where clients send gzipped data without proper headers
if has_gzip_magic_header(&body) {
tracing::debug!("Auto-detected gzip compression from magic bytes");
inc(
FLAG_REQUEST_KLUDGE_COUNTER,
&[("type".to_string(), "auto_detected_gzip".to_string())],
1,
);
return decompress_gzip(body);
}
// No compression detected
Ok(body)
}
fn decompress_gzip(compressed: Bytes) -> Result<Bytes, FlagError> {
common_compression::decompress_gzip_capped(&compressed, MAX_FLAGS_DECOMPRESSED_BYTES)
.map(Bytes::from)
.map_err(|e| {
if matches!(
e,
common_compression::CompressionError::OutputTooLarge { .. }
) {
inc(FLAG_GZIP_OUTPUT_EXCEEDED_COUNTER, &[], 1);
} else {
tracing::warn!("gzip decompression failed: {}", e);
}
FlagError::from(e)
})
}
fn decode_base64(body: Bytes) -> Result<Bytes, FlagError> {
// Convert to string and apply URL decoding like base64_decode in Python decide
let body_str = String::from_utf8_lossy(&body);
let url_decoded = percent_decode(body_str.as_bytes())
.decode_utf8()
.map_err(|e| {
tracing::warn!("Failed to URL decode base64 data: {}", e);
FlagError::RequestDecodingError(format!("Failed to URL decode: {e}"))
})?;
// Remove whitespace and add padding if necessary
let mut cleaned = url_decoded.replace(" ", "");
let padding_needed = cleaned.len() % 4;
if padding_needed > 0 {
cleaned.push_str(&"=".repeat(4 - padding_needed));
}
let decoded = general_purpose::STANDARD.decode(cleaned).map_err(|e| {
tracing::warn!("Base64 decoding error: {}", e);
FlagError::RequestDecodingError(format!("Base64 decoding error: {e}"))
})?;
Ok(Bytes::from(decoded))
}
/// Parse a request body that has already been gzip-decompressed (if it was
/// gzipped). Falls back to base64-decoding when the body is not direct JSON
/// — some SDKs send base64-wrapped JSON via `application/json` rather than
/// the form-urlencoded path.
///
/// Returns the parsed [`FlagRequest`] along with the bytes that successfully
/// parsed: the input bytes when direct JSON parsing wins, or the
/// base64-decoded bytes when the fallback wins.
pub fn try_parse_with_fallbacks(
body: Bytes,
user_agent: Option<&str>,
) -> Result<(FlagRequest, Bytes), FlagError> {
// Strategy 1: Try parsing as JSON directly
if let Ok(request) = FlagRequest::from_bytes(body.clone()) {
return Ok((request, body));
}
// Strategy 2: Try base64 decode then JSON
// Even if compression is not specified, we still try to decode it as base64
let client_type = UserAgentInfo::client_type_label_from_raw(user_agent);
tracing::warn!(
client_type = client_type,
"Direct JSON parsing failed, trying base64 decode fallback"
);
match decode_base64(body.clone()) {
Ok(decoded) => match FlagRequest::from_bytes(decoded.clone()) {
Ok(request) => {
inc(
FLAG_REQUEST_KLUDGE_COUNTER,
&[
("type".to_string(), "base64_fallback_success".to_string()),
("client_type".to_string(), client_type.to_string()),
],
1,
);
tracing::info!(
client_type = client_type,
"Successfully parsed request after base64 fallback decoding"
);
return Ok((request, decoded));
}
Err(e) => {
tracing::warn!("Base64 decode succeeded but JSON parsing failed: {}", e);
}
},
Err(e) => {
tracing::warn!("Base64 decode failed: {}", e);
}
}
Err(FlagError::RequestDecodingError("invalid JSON".to_string()))
}
pub fn decode_form_data(
body: Bytes,
compression: Option<Compression>,
user_agent: Option<&str>,
) -> Result<(FlagRequest, Bytes), FlagError> {
// Convert bytes to string first so we can manipulate it
let form_data = String::from_utf8(body.to_vec()).map_err(|e| {
tracing::warn!("Invalid UTF-8 in form data: {}", e);
FlagError::RequestDecodingError("Invalid UTF-8 in form data".into())
})?;
// URL decode the string if needed
let decoded_form = percent_decode(form_data.as_bytes())
.decode_utf8()
.map_err(|e| {
tracing::warn!("Failed to URL decode form data: {}", e);
FlagError::RequestDecodingError("Failed to URL decode form data".into())
})?;
// Extract base64 part, handling both with and without 'data=' prefix
// see https://github.com/PostHog/posthog/blob/master/posthog/utils.py#L693-L699
let base64_str = if decoded_form.starts_with("data=") {
decoded_form.split('=').nth(1).unwrap_or("")
} else {
// Count how often we receive base64 data without the 'data=' prefix
// Include client_type to help identify which SDKs are sending malformed data
let client_type = UserAgentInfo::client_type_label_from_raw(user_agent);
inc(
FLAG_REQUEST_KLUDGE_COUNTER,
&[
("type".to_string(), "missing_data_prefix".to_string()),
("client_type".to_string(), client_type.to_string()),
],
1,
);
&decoded_form
};
// Remove whitespace and add padding if necessary
// https://github.com/PostHog/posthog/blob/master/posthog/utils.py#L701-L705
let mut cleaned_base64 = base64_str.replace(' ', "");
let padding_needed = cleaned_base64.len() % 4;
if padding_needed > 0 {
inc(
FLAG_REQUEST_KLUDGE_COUNTER,
&[("type".to_string(), "padding_needed".to_string())],
1,
);
cleaned_base64.push_str(&"=".repeat(4 - padding_needed));
}
// Handle compression if specified (we don't support gzip for form-urlencoded data)
let decoded = match compression {
Some(Compression::Gzip) => {
tracing::warn!("Gzip compression not supported for form-urlencoded data");
return Err(FlagError::RequestDecodingError(
"Gzip compression not supported for form-urlencoded data".into(),
));
}
Some(Compression::Base64) | None => decode_base64(Bytes::from(cleaned_base64))?,
Some(Compression::Unsupported) => {
tracing::warn!("Unsupported compression type for form-urlencoded data");
return Err(FlagError::RequestDecodingError(
"Unsupported compression type".into(),
));
}
};
// Convert to UTF-8 string with utf8_lossy to handle invalid UTF-8 sequences
// this is equivalent to using Python's `surrogatepass`, since it just replaces
// unparseable characters with the Unicode replacement character (U+FFFD) instead of failing to decode the request
// at all.
let json_str = {
let lossy_str = String::from_utf8_lossy(&decoded);
// Count how often we receive base64 data with invalid UTF-8 sequences
if lossy_str.contains('\u{FFFD}') {
inc(
FLAG_REQUEST_KLUDGE_COUNTER,
&[("type".to_string(), "lossy_utf8".to_string())],
1,
);
}
lossy_str.into_owned()
};
// Parse JSON into FlagRequest. Return the decoded JSON bytes alongside
// the parsed request so callers (e.g. body logging) don't have to redo
// the URL-decode + base64-decode dance to recover them.
let json_bytes = Bytes::from(json_str.into_bytes());
let request = serde_json::from_slice(&json_bytes).map_err(|e| {
tracing::warn!("failed to parse JSON: {}", e);
FlagError::RequestDecodingError("invalid JSON structure".into())
})?;
Ok((request, json_bytes))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::api::types::{Compression, FlagsQueryParams};
use axum::http::HeaderMap;
fn create_gzipped_json(json_data: &str) -> Bytes {
let compressed = common_compression::compress_gzip(json_data.as_bytes()).unwrap();
Bytes::from(compressed)
}
#[test]
fn test_gzip_auto_detection_from_magic_bytes() {
let json_data = r#"{"distinct_id": "test", "token": "test_token"}"#;
let gzipped_body = create_gzipped_json(json_data);
// Verify magic bytes are present
assert_eq!(gzipped_body[0], 0x1f);
assert_eq!(gzipped_body[1], 0x8b);
let headers = HeaderMap::new();
let query = FlagsQueryParams::default(); // No compression specified
let result = decode_request(&headers, gzipped_body, &query);
assert!(result.is_ok());
let (request, decoded) = result.unwrap();
assert_eq!(request.distinct_id, Some("test".to_string()));
assert_eq!(request.token, Some("test_token".to_string()));
// Decoded body is the post-gzip JSON, not the gzipped bytes.
assert!(decoded.starts_with(b"{"));
assert!(decoded.windows(11).any(|w| w == b"distinct_id"));
}
#[test]
fn test_gzip_detection_from_content_encoding_header() {
let json_data = r#"{"distinct_id": "test", "token": "test_token"}"#;
let gzipped_body = create_gzipped_json(json_data);
let mut headers = HeaderMap::new();
headers.insert("content-encoding", "gzip".parse().unwrap());
let query = FlagsQueryParams::default(); // No compression specified
let result = decode_request(&headers, gzipped_body, &query);
assert!(result.is_ok());
let (request, _decoded) = result.unwrap();
assert_eq!(request.distinct_id, Some("test".to_string()));
assert_eq!(request.token, Some("test_token".to_string()));
}
#[test]
fn test_gzip_bomb_rejected_with_payload_too_large() {
// Compress 8 MiB of zeros — gzip's redundant-input case where the
// decompressed size dwarfs the compressed size. Decompresses to
// 8 MiB, which is well over MAX_FLAGS_DECOMPRESSED_BYTES (4 MiB).
let bomb = vec![0u8; 8 * 1024 * 1024];
let compressed = common_compression::compress_gzip(&bomb).unwrap();
assert!(
compressed.len() < 100_000,
"bomb should compress small (got {} bytes)",
compressed.len()
);
let mut headers = HeaderMap::new();
headers.insert("content-encoding", "gzip".parse().unwrap());
let query = FlagsQueryParams::default();
let result = decode_request(&headers, Bytes::from(compressed), &query);
match result {
Err(FlagError::PayloadTooLarge {
decompressed,
limit,
}) => {
assert_eq!(limit, MAX_FLAGS_DECOMPRESSED_BYTES);
assert!(
decompressed > MAX_FLAGS_DECOMPRESSED_BYTES,
"decompressed size {decompressed} should exceed cap {MAX_FLAGS_DECOMPRESSED_BYTES}"
);
}
other => panic!("expected PayloadTooLarge, got {other:?}"),
}
}
#[test]
fn test_gzip_under_cap_still_works() {
// A 1 MiB body is well over typical /flags traffic but still inside
// the 4 MiB cap, so it must round-trip cleanly.
let mut large_payload = String::from(r#"{"distinct_id": "u", "token": "t", "padding": ""#);
large_payload.push_str(&"a".repeat(1024 * 1024));
large_payload.push_str(r#""}"#);
let gzipped = create_gzipped_json(&large_payload);
let mut headers = HeaderMap::new();
headers.insert("content-encoding", "gzip".parse().unwrap());
let query = FlagsQueryParams::default();
let result = decode_request(&headers, gzipped, &query);
assert!(
result.is_ok(),
"1 MiB request body should decode under the 4 MiB cap, got {:?}",
result.err()
);
}
#[test]
fn test_explicit_gzip_compression_parameter() {
let json_data = r#"{"distinct_id": "test", "token": "test_token"}"#;
let gzipped_body = create_gzipped_json(json_data);
let headers = HeaderMap::new();
let query = FlagsQueryParams {
compression: Some(Compression::Gzip),
..Default::default()
};
let result = decode_request(&headers, gzipped_body, &query);
assert!(result.is_ok());
let (request, _decoded) = result.unwrap();
assert_eq!(request.distinct_id, Some("test".to_string()));
assert_eq!(request.token, Some("test_token".to_string()));
}
#[test]
fn test_uncompressed_json_still_works() {
let json_data = r#"{"distinct_id": "test", "token": "test_token"}"#;
let body = Bytes::from(json_data);
let headers = HeaderMap::new();
let query = FlagsQueryParams::default();
let result = decode_request(&headers, body.clone(), &query);
assert!(result.is_ok());
let (request, decoded) = result.unwrap();
assert_eq!(request.distinct_id, Some("test".to_string()));
assert_eq!(request.token, Some("test_token".to_string()));
// Plain-JSON path returns the input bytes unchanged.
assert_eq!(decoded, body);
}
#[test]
fn test_base64_with_url_encoding() {
// This test verifies the fix for URL-encoded base64 data
// Base64 with padding: eyJ0b2tlbiI6ICJ0ZXN0IiwgImRpc3RpbmN0X2lkIjogInVzZXIifQo=
// URL-encoded (= becomes %3D): eyJ0b2tlbiI6ICJ0ZXN0IiwgImRpc3RpbmN0X2lkIjogInVzZXIifQo%3D
let url_encoded_base64 = "eyJ0b2tlbiI6ICJ0ZXN0IiwgImRpc3RpbmN0X2lkIjogInVzZXIifQo%3D";
let body = Bytes::from(url_encoded_base64);
let headers = HeaderMap::new();
let query = FlagsQueryParams {
compression: Some(Compression::Base64),
..Default::default()
};
let result = decode_request(&headers, body, &query);
assert!(result.is_ok());
let (request, decoded) = result.unwrap();
assert_eq!(request.distinct_id, Some("user".to_string()));
assert_eq!(request.token, Some("test".to_string()));
// Base64 fallback returns the *decoded* JSON bytes, not the base64 string.
assert!(decoded.starts_with(b"{"));
assert!(!decoded.windows(7).any(|w| w == b"eyJ0b2t"));
}
#[test]
fn test_extract_token_with_token_field() {
let json_data = r#"{"token": "test_token_123", "distinct_id": "user"}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, Some("test_token_123".to_string()));
}
#[test]
fn test_extract_token_with_api_key_field() {
let json_data = r#"{"api_key": "phc_test123", "distinct_id": "user"}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, Some("phc_test123".to_string()));
}
#[test]
fn test_extract_token_with_dollar_token_field() {
let json_data = r#"{"$token": "test_token", "distinct_id": "user"}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, Some("test_token".to_string()));
}
#[test]
fn test_extract_token_prefers_token_over_api_key() {
let json_data = r#"{"token": "token_value", "api_key": "api_key_value"}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, Some("token_value".to_string()));
}
#[test]
fn test_extract_token_with_empty_token() {
let json_data = r#"{"token": "", "distinct_id": "user"}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, None);
}
#[test]
fn test_extract_token_with_no_token_field() {
let json_data = r#"{"distinct_id": "user", "properties": {}}"#;
let body = Bytes::from(json_data);
let token = extract_token(&body);
assert_eq!(token, None);
}
#[test]
fn test_extract_token_with_invalid_json() {
let body = Bytes::from("not valid json {{{");
let token = extract_token(&body);
assert_eq!(token, None);
}
#[test]
fn test_extract_token_with_empty_body() {
let body = Bytes::from("");
let token = extract_token(&body);
assert_eq!(token, None);
}
mod client_type_label_tests {
use super::*;
use rstest::rstest;
#[rstest]
#[case(Some("posthog-js/1.88.0"), "posthog-js")]
#[case(Some("posthog-android/3.1.0"), "posthog-android")]
#[case(Some("posthog-ios/3.0.0"), "posthog-ios")]
#[case(Some("posthog-react-native/2.5.0"), "posthog-react-native")]
#[case(Some("posthog-flutter/4.0.0"), "posthog-flutter")]
#[case(Some("posthog-kmp/0.6.0"), "posthog-kmp")]
#[case(Some("posthog-unity/4.5.0"), "posthog-unity")]
#[case(Some("posthog-python/1.4.0"), "posthog-python")]
#[case(Some("posthog-python-mcp/0.1.0"), "posthog-python-mcp")]
#[case(Some("posthog-ruby/2.0.0"), "posthog-ruby")]
#[case(Some("posthog-rails/3.18.0"), "posthog-rails")]
#[case(Some("posthog-ruby2.0.0"), "posthog-ruby")]
#[case(Some("posthog-php/3.0.0"), "posthog-php")]
#[case(Some("posthog-laravel/0.1.0"), "posthog-laravel")]
#[case(Some("posthog-java/1.0.0"), "posthog-java")]
#[case(Some("posthog-go/0.1.0"), "posthog-go")]
#[case(Some("posthog-node/2.2.0"), "posthog-node")]
#[case(Some("posthog-node-mcp/0.7.0"), "posthog-node-mcp")]
#[case(Some("posthog-edge/2.2.0"), "posthog-edge")]
#[case(Some("posthog-convex/0.2.0"), "posthog-convex")]
#[case(Some("posthog-dotnet/1.0.0"), "posthog-dotnet")]
#[case(Some("posthog-aspnetcore/1.0.0"), "posthog-aspnetcore")]
#[case(Some("posthog-elixir/0.2.0"), "posthog-elixir")]
#[case(Some("posthog-rs/0.10.0"), "posthog-rs")]
#[case(Some("posthog-server/1.0.0"), "posthog-server")]
#[case(
Some("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"),
"browser"
)]
#[case(
Some("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Chrome/91.0.4472.124"),
"browser"
)]
#[case(Some("Mozilla/5.0 (X11; Linux x86_64) Firefox/89.0"), "browser")]
#[case(Some("curl/7.68.0"), "curl")]
#[case(Some("python-requests/2.28.0"), "python-requests")]
#[case(Some("custom-client/1.0"), "other")]
#[case(Some(""), "unknown")]
#[case(None, "unknown")]
fn test_client_type_label(#[case] user_agent: Option<&str>, #[case] expected: &str) {
assert_eq!(
UserAgentInfo::client_type_label_from_raw(user_agent),
expected
);
}
}
}