Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(capture): count legacy events that exceed capture v1 field limits (#120600)

Eli Reismanc5c19878146d
diff --git a/rust/capture/src/events/analytics.rs b/rust/capture/src/events/analytics.rs
index ae7f48142165..aee9c014b4ec 100644
--- a/rust/capture/src/events/analytics.rs
+++ b/rust/capture/src/events/analytics.rs
@@ -10,7 +10,7 @@ use chrono::DateTime;
use common_ingestion_warnings::{
WarningEmitter, CAPTURE_LEGACY_ANALYTICS, CAPTURE_LEGACY_RATE_LIMIT,
};
-use common_types::{CapturedEvent, RawEvent};
+use common_types::{CapturedEvent, ExtractedDistinctId, RawEvent};
use limiters::token_dropper::TokenDropper;
use metrics::{counter, histogram};
use serde_json;
@@ -30,13 +30,19 @@ use crate::{
legacy::{emit_processing_abort_warning, request_context},
},
outputs::OutputRegistry,
- prometheus::{report_clock_skew, report_dropped_events, report_timestamp_path},
+ prometheus::{
+ report_clock_skew, report_dropped_events, report_over_v1_limit, report_timestamp_path,
+ },
router,
utils::uuid_v7_from_datetime,
v0_request::{
exceeds_max_ai_event_bytes, DataType, OverflowReason, ProcessedEvent,
ProcessedEventMetadata, ProcessingContext,
},
+ v1::{
+ analytics::constants::{CAPTURE_V1_DISTINCT_ID_MAX_SIZE, CAPTURE_V1_MAX_EVENT_NAME_LENGTH},
+ error::Error as V1Error,
+ },
};
/// Property keys the heatmap pipeline reads from a redirected event. The
@@ -116,6 +122,16 @@ fn create_heatmap_redirect(
process_single_event(&heatmap_event, historical_cfg, context).map(Some)
}
+/// Count fields this event keeps on legacy capture but that capture v1 rejects, measured in UTF-8 bytes like v1.
+fn report_v1_limit_overruns(event_name: &str, distinct_id: &ExtractedDistinctId) {
+ if event_name.len() > CAPTURE_V1_MAX_EVENT_NAME_LENGTH {
+ report_over_v1_limit(V1Error::EventNameTooLong.tag());
+ }
+ if distinct_id.trimmed_bytes > CAPTURE_V1_DISTINCT_ID_MAX_SIZE {
+ report_over_v1_limit(V1Error::DistinctIdTooLarge.tag());
+ }
+}
+
/// Process a single analytics event from RawEvent to ProcessedEvent.
#[instrument(skip_all, fields(event_name, request_id))]
pub fn process_single_event(
@@ -176,6 +192,7 @@ pub fn process_single_event(
let extracted_distinct_id = event
.extract_distinct_id_checked()
.ok_or(CaptureError::MissingDistinctId)?;
+ report_v1_limit_overruns(&event_name, &extracted_distinct_id);
let mut metadata = ProcessedEventMetadata {
data_type,
@@ -3837,6 +3854,88 @@ mod tests {
}
}
+ /// Counter value for `capture_events_over_v1_limits_total` at the given `cause`.
+ fn over_v1_limit_count(
+ snapshotter: &metrics_util::debugging::Snapshotter,
+ cause: &str,
+ ) -> Option<u64> {
+ use metrics_util::debugging::DebugValue;
+
+ snapshotter
+ .snapshot()
+ .into_vec()
+ .into_iter()
+ .find_map(|(key, _, _, value)| {
+ if key.key().name() != crate::prometheus::CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL {
+ return None;
+ }
+ let matches = key
+ .key()
+ .labels()
+ .any(|l| l.key() == "cause" && l.value() == cause);
+ match value {
+ DebugValue::Counter(v) if matches => Some(v),
+ _ => None,
+ }
+ })
+ }
+
+ #[rstest]
+ #[case::short_fields("test_event", "user-1".to_string(), None, None)]
+ #[case::distinct_id_at_limit("test_event", "x".repeat(200), None, None)]
+ #[case::distinct_id_over_limit_and_shortened("test_event", "x".repeat(201), None, Some(1))]
+ #[case::distinct_id_over_limit_in_bytes_only("test_event", "é".repeat(101), None, Some(1))]
+ #[case::distinct_id_fits_after_trimming(
+ "test_event",
+ format!(" {} ", "x".repeat(199)),
+ None,
+ None
+ )]
+ #[case::event_name_at_limit(&"e".repeat(200), "user-1".to_string(), None, None)]
+ #[case::event_name_over_limit(&"e".repeat(201), "user-1".to_string(), Some(1), None)]
+ #[case::event_name_over_limit_in_bytes_only(&"é".repeat(101), "user-1".to_string(), Some(1), None)]
+ #[case::both_over_limit(&"e".repeat(201), "x".repeat(201), Some(1), Some(1))]
+ fn counts_events_capture_v1_would_reject(
+ #[case] event_name: &str,
+ #[case] distinct_id: String,
+ #[case] expected_name_too_long: Option<u64>,
+ #[case] expected_distinct_id_too_large: Option<u64>,
+ ) {
+ let recorder = metrics_util::debugging::DebuggingRecorder::new();
+ let snapshotter = recorder.snapshotter();
+ let _metrics_guard = metrics::set_default_local_recorder(&recorder);
+
+ let now = DateTime::parse_from_rfc3339("2023-01-01T12:00:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let mut event = create_test_event_with_name(
+ event_name,
+ Some("2023-01-01T11:00:00Z".to_string()),
+ None,
+ None,
+ );
+ event
+ .properties
+ .insert("distinct_id".to_string(), json!(distinct_id));
+
+ // Legacy capture still accepts the event; the counter only sizes what v1 would drop.
+ process_single_event(
+ &event,
+ router::HistoricalConfig::new(false, 1),
+ &create_test_context(now, None),
+ )
+ .unwrap();
+
+ assert_eq!(
+ over_v1_limit_count(&snapshotter, "event_name_too_long"),
+ expected_name_too_long
+ );
+ assert_eq!(
+ over_v1_limit_count(&snapshotter, "distinct_id_too_large"),
+ expected_distinct_id_too_large
+ );
+ }
+
fn event_with_distinct_id(distinct_id: &str) -> RawEvent {
let mut event = create_test_event(Some("2023-01-01T11:00:00Z".to_string()), None, None);
event
diff --git a/rust/capture/src/events/recordings.rs b/rust/capture/src/events/recordings.rs
index 5f8fdd38d09b..7744a432b8a5 100644
--- a/rust/capture/src/events/recordings.rs
+++ b/rust/capture/src/events/recordings.rs
@@ -194,6 +194,7 @@ impl RawRecording {
if distinct_id.is_empty() {
return None;
}
+ let trimmed_bytes = distinct_id.trim().len();
// Byte length bounds char count, so ids within 200 bytes skip the
// char-count pass entirely on the hot path.
@@ -201,6 +202,7 @@ impl RawRecording {
return Some(ExtractedDistinctId {
value: distinct_id,
truncated_from_chars: None,
+ trimmed_bytes,
});
}
@@ -209,12 +211,14 @@ impl RawRecording {
return Some(ExtractedDistinctId {
value: distinct_id,
truncated_from_chars: None,
+ trimmed_bytes,
});
}
Some(ExtractedDistinctId {
value: distinct_id.chars().take(200).collect(),
truncated_from_chars: Some(char_count),
+ trimmed_bytes,
})
}
diff --git a/rust/capture/src/prometheus.rs b/rust/capture/src/prometheus.rs
index 8297c3639452..dbf020dc96e1 100644
--- a/rust/capture/src/prometheus.rs
+++ b/rust/capture/src/prometheus.rs
@@ -6,6 +6,7 @@ use metrics::counter;
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
pub const CAPTURE_EVENTS_DROPPED_TOTAL: &str = "capture_events_dropped_total";
+pub const CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL: &str = "capture_events_over_v1_limits_total";
pub const CAPTURE_TIMESTAMP_PATH_TOTAL: &str = "capture_timestamp_path_total";
pub const CAPTURE_STORED_VS_CLIENT_CAPTURE_SECONDS: &str =
@@ -17,6 +18,11 @@ pub fn report_dropped_events(cause: &'static str, quantity: u64) {
counter!(CAPTURE_EVENTS_DROPPED_TOTAL, "cause" => cause).increment(quantity);
}
+/// Counts a legacy event that capture v1 would drop; `cause` is the v1 per-event detail tag.
+pub fn report_over_v1_limit(cause: &'static str) {
+ counter!(CAPTURE_EVENTS_OVER_V1_LIMITS_TOTAL, "cause" => cause).increment(1);
+}
+
pub fn report_overflow_partition(quantity: u64) {
counter!("capture_partition_key_capacity_exceeded_total").increment(quantity);
}
diff --git a/rust/capture/src/v1/analytics/constants.rs b/rust/capture/src/v1/analytics/constants.rs
index e0523558abdc..c42f263231c2 100644
--- a/rust/capture/src/v1/analytics/constants.rs
+++ b/rust/capture/src/v1/analytics/constants.rs
@@ -90,10 +90,10 @@ pub(super) const DETAIL_AI_EVENT_TOO_BIG: &str = "ai_event_too_big";
// ---------------------------------------------------------------------------
/// Maximum allowed length for an event name; longer names are rejected.
-pub(super) const CAPTURE_V1_MAX_EVENT_NAME_LENGTH: usize = 200;
+pub(crate) const CAPTURE_V1_MAX_EVENT_NAME_LENGTH: usize = 200;
/// Maximum allowed length for a distinct_id value; longer IDs are rejected.
-pub(super) const CAPTURE_V1_DISTINCT_ID_MAX_SIZE: usize = 200;
+pub(crate) const CAPTURE_V1_DISTINCT_ID_MAX_SIZE: usize = 200;
// ---------------------------------------------------------------------------
// Illegal distinct_id values
diff --git a/rust/common/types/src/event.rs b/rust/common/types/src/event.rs
index 8883d0a52969..fa1b05c61e2a 100644
--- a/rust/common/types/src/event.rs
+++ b/rust/common/types/src/event.rs
@@ -446,6 +446,8 @@ pub struct ExtractedDistinctId {
/// Char count of the original id when it exceeded the cap and `value` was
/// cut down; `None` when the id was ingested unmodified.
pub truncated_from_chars: Option<usize>,
+ /// UTF-8 byte length of the original id after trimming whitespace, the size capture v1 limits.
+ pub trimmed_bytes: usize,
}
impl RawEvent {
@@ -498,6 +500,7 @@ impl RawEvent {
if trimmed.len() != distinct_id.len() {
counter!("capture_distinct_id_has_whitespace_total").increment(1);
}
+ let trimmed_bytes = trimmed.len();
// Byte length bounds char count, so ids within 200 bytes skip the
// char-count pass entirely on the hot path.
@@ -505,6 +508,7 @@ impl RawEvent {
return Some(ExtractedDistinctId {
value: distinct_id,
truncated_from_chars: None,
+ trimmed_bytes,
});
}
@@ -516,12 +520,14 @@ impl RawEvent {
return Some(ExtractedDistinctId {
value: distinct_id,
truncated_from_chars: None,
+ trimmed_bytes,
});
}
Some(ExtractedDistinctId {
value: distinct_id.chars().take(200).collect(),
truncated_from_chars: Some(char_count),
+ trimmed_bytes,
})
}