:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(capture): count legacy events that exceed capture v1 field limits (#120600)
Eli Reismanc5c19878146d
diff --git a/rust/capture/src/events/analytics.rs b/rust/capture/src/events/analytics.rsindex ae7f48142165..aee9c014b4ec 100644--- a/rust/capture/src/events/analytics.rs+++ b/rust/capture/src/events/analytics.rs@@ -10,7 +10,7 @@ use chrono::DateTime;use common_ingestion_warnings::{WarningEmitter, CAPTURE_LEGACY_ANALYTICS, CAPTURE_LEGACY_RATE_LIMIT,};-use common_types::{CapturedEvent, RawEvent};+use common_types::{CapturedEvent, ExtractedDistinctId, RawEvent};use limiters::token_dropper::TokenDropper;use metrics::{counter, histogram};use serde_json;@@ -30,13 +30,19 @@ use crate::{legacy::{emit_processing_abort_warning, request_context},},outputs::OutputRegistry,- prometheus::{report_clock_skew, report_dropped_events, report_timestamp_path},+ prometheus::{+ report_clock_skew, report_dropped_events, report_over_v1_limit, report_timestamp_path,+ },router,utils::uuid_v7_from_datetime,v0_request::{exceeds_max_ai_event_bytes, DataType, OverflowReason, ProcessedEvent,ProcessedEventMetadata, ProcessingContext,},+ v1::{+ analytics::constants::{CAPTURE_V1_DISTINCT_ID_MAX_SIZE, CAPTURE_V1_MAX_EVENT_NAME_LENGTH},+ error::Error as V1Error,+ },};/// Property keys the heatmap pipeline reads from a redirected event. The@@ -116,6 +122,16 @@ fn create_heatmap_redirect(process_single_event(&heatmap_event, historical_cfg, context).map(Some)}+/// Count fields this event keeps on legacy capture but that capture v1 rejects, measured in UTF-8 bytes like v1.+fn report_v1_limit_overruns(event_name: &str, distinct_id: &ExtractedDistinctId) {+ if event_name.len() > CAPTURE_V1_MAX_EVENT_NAME_LENGTH {+ report_over_v1_limit(V1Error::EventNameTooLong.tag());+ }+ if distinct_id.trimmed_bytes > CAPTURE_V1_DISTINCT_ID_MAX_SIZE {+ report_over_v1_limit(V1Error::DistinctIdTooLarge.tag());+ }+}+/// Process a single analytics event from RawEvent to ProcessedEvent.#[instrument(skip_all, fields(event_name, request_id))]pub fn process_single_event(@@ -176,6 +192,7 @@ pub fn process_single_event(let extracted_distinct_id = event.extract_distinct_id_checked().ok_or(CaptureError::MissingDistinctId)?;+ report_v1_limit_overruns(&event_name, &extracted_distinct_id);let mut metadata = ProcessedEventMetadata {data_type,@@ -3837,6 +3854,88 @@ mod tests {}}+ /// Counter value for `capture_events_over_v1_limits_total` at the given `cause`.+ fn over_v1_limit_count(+ snapshotter: &metrics_util::debugging::Snapshotter,+ cause: &str,+ ) -> Option<u64> {+ use metrics_util::debugging::DebugValue;++ snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .find_map(|(key, _, _, value)| {+ if key.key().name() != crate::prometheus::CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL {+ return None;+ }+ let matches = key+ .key()+ .labels()+ .any(|l| l.key() == "cause" && l.value() == cause);+ match value {+ DebugValue::Counter(v) if matches => Some(v),+ _ => None,+ }+ })+ }++ #[rstest]+ #[case::short_fields("test_event", "user-1".to_string(), None, None)]+ #[case::distinct_id_at_limit("test_event", "x".repeat(200), None, None)]+ #[case::distinct_id_over_limit_and_shortened("test_event", "x".repeat(201), None, Some(1))]+ #[case::distinct_id_over_limit_in_bytes_only("test_event", "é".repeat(101), None, Some(1))]+ #[case::distinct_id_fits_after_trimming(+ "test_event",+ format!(" {} ", "x".repeat(199)),+ None,+ None+ )]+ #[case::event_name_at_limit(&"e".repeat(200), "user-1".to_string(), None, None)]+ #[case::event_name_over_limit(&"e".repeat(201), "user-1".to_string(), Some(1), None)]+ #[case::event_name_over_limit_in_bytes_only(&"é".repeat(101), "user-1".to_string(), Some(1), None)]+ #[case::both_over_limit(&"e".repeat(201), "x".repeat(201), Some(1), Some(1))]+ fn counts_events_capture_v1_would_reject(+ #[case] event_name: &str,+ #[case] distinct_id: String,+ #[case] expected_name_too_long: Option<u64>,+ #[case] expected_distinct_id_too_large: Option<u64>,+ ) {+ let recorder = metrics_util::debugging::DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _metrics_guard = metrics::set_default_local_recorder(&recorder);++ let now = DateTime::parse_from_rfc3339("2023-01-01T12:00:00Z")+ .unwrap()+ .with_timezone(&Utc);+ let mut event = create_test_event_with_name(+ event_name,+ Some("2023-01-01T11:00:00Z".to_string()),+ None,+ None,+ );+ event+ .properties+ .insert("distinct_id".to_string(), json!(distinct_id));++ // Legacy capture still accepts the event; the counter only sizes what v1 would drop.+ process_single_event(+ &event,+ router::HistoricalConfig::new(false, 1),+ &create_test_context(now, None),+ )+ .unwrap();++ assert_eq!(+ over_v1_limit_count(&snapshotter, "event_name_too_long"),+ expected_name_too_long+ );+ assert_eq!(+ over_v1_limit_count(&snapshotter, "distinct_id_too_large"),+ expected_distinct_id_too_large+ );+ }+fn event_with_distinct_id(distinct_id: &str) -> RawEvent {let mut event = create_test_event(Some("2023-01-01T11:00:00Z".to_string()), None, None);eventdiff --git a/rust/capture/src/events/recordings.rs b/rust/capture/src/events/recordings.rsindex 5f8fdd38d09b..7744a432b8a5 100644--- a/rust/capture/src/events/recordings.rs+++ b/rust/capture/src/events/recordings.rs@@ -194,6 +194,7 @@ impl RawRecording {if distinct_id.is_empty() {return None;}+ let trimmed_bytes = distinct_id.trim().len();// Byte length bounds char count, so ids within 200 bytes skip the// char-count pass entirely on the hot path.@@ -201,6 +202,7 @@ impl RawRecording {return Some(ExtractedDistinctId {value: distinct_id,truncated_from_chars: None,+ trimmed_bytes,});}@@ -209,12 +211,14 @@ impl RawRecording {return Some(ExtractedDistinctId {value: distinct_id,truncated_from_chars: None,+ trimmed_bytes,});}Some(ExtractedDistinctId {value: distinct_id.chars().take(200).collect(),truncated_from_chars: Some(char_count),+ trimmed_bytes,})}diff --git a/rust/capture/src/prometheus.rs b/rust/capture/src/prometheus.rsindex 8297c3639452..dbf020dc96e1 100644--- a/rust/capture/src/prometheus.rs+++ b/rust/capture/src/prometheus.rs@@ -6,6 +6,7 @@ use metrics::counter;use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};pub const CAPTURE_EVENTS_DROPPED_TOTAL: &str = "capture_events_dropped_total";+pub const CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL: &str = "capture_events_over_v1_limits_total";pub const CAPTURE_TIMESTAMP_PATH_TOTAL: &str = "capture_timestamp_path_total";pub const CAPTURE_STORED_VS_CLIENT_CAPTURE_SECONDS: &str =@@ -17,6 +18,11 @@ pub fn report_dropped_events(cause: &'static str, quantity: u64) {counter!(CAPTURE_EVENTS_DROPPED_TOTAL, "cause" => cause).increment(quantity);}+/// Counts a legacy event that capture v1 would drop; `cause` is the v1 per-event detail tag.+pub fn report_over_v1_limit(cause: &'static str) {+ counter!(CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL, "cause" => cause).increment(1);+}+pub fn report_overflow_partition(quantity: u64) {counter!("capture_partition_key_capacity_exceeded_total").increment(quantity);}diff --git a/rust/capture/src/v1/analytics/constants.rs b/rust/capture/src/v1/analytics/constants.rsindex e0523558abdc..c42f263231c2 100644--- a/rust/capture/src/v1/analytics/constants.rs+++ b/rust/capture/src/v1/analytics/constants.rs@@ -90,10 +90,10 @@ pub(super) const DETAIL_AI_EVENT_TOO_BIG: &str = "ai_event_too_big";// ---------------------------------------------------------------------------/// Maximum allowed length for an event name; longer names are rejected.-pub(super) const CAPTURE_V1_MAX_EVENT_NAME_LENGTH: usize = 200;+pub(crate) const CAPTURE_V1_MAX_EVENT_NAME_LENGTH: usize = 200;/// Maximum allowed length for a distinct_id value; longer IDs are rejected.-pub(super) const CAPTURE_V1_DISTINCT_ID_MAX_SIZE: usize = 200;+pub(crate) const CAPTURE_V1_DISTINCT_ID_MAX_SIZE: usize = 200;// ---------------------------------------------------------------------------// Illegal distinct_id valuesdiff --git a/rust/common/types/src/event.rs b/rust/common/types/src/event.rsindex 8883d0a52969..fa1b05c61e2a 100644--- a/rust/common/types/src/event.rs+++ b/rust/common/types/src/event.rs@@ -446,6 +446,8 @@ pub struct ExtractedDistinctId {/// Char count of the original id when it exceeded the cap and `value` was/// cut down; `None` when the id was ingested unmodified.pub truncated_from_chars: Option<usize>,+ /// UTF-8 byte length of the original id after trimming whitespace, the size capture v1 limits.+ pub trimmed_bytes: usize,}impl RawEvent {@@ -498,6 +500,7 @@ impl RawEvent {if trimmed.len() != distinct_id.len() {counter!("capture_distinct_id_has_whitespace_total").increment(1);}+ let trimmed_bytes = trimmed.len();// Byte length bounds char count, so ids within 200 bytes skip the// char-count pass entirely on the hot path.@@ -505,6 +508,7 @@ impl RawEvent {return Some(ExtractedDistinctId {value: distinct_id,truncated_from_chars: None,+ trimmed_bytes,});}@@ -516,12 +520,14 @@ impl RawEvent {return Some(ExtractedDistinctId {value: distinct_id,truncated_from_chars: None,+ trimmed_bytes,});}Some(ExtractedDistinctId {value: distinct_id.chars().take(200).collect(),truncated_from_chars: Some(char_count),+ trimmed_bytes,})}