:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(capture): keep the first copy of a duplicate v1 event uuid (#122305)
Eli Reisman55c57e6ff550
diff --git a/rust/batch-import-worker/src/emit/capture.rs b/rust/batch-import-worker/src/emit/capture.rsindex 909a739c2be4..0ecbe0cf1731 100644--- a/rust/batch-import-worker/src/emit/capture.rs+++ b/rust/batch-import-worker/src/emit/capture.rs@@ -25,9 +25,9 @@ pub struct CaptureTransaction<'a> {send_rate: u64,start: Instant,events: Mutex<Vec<Event>>,- // Capture's V1 endpoint rejects a whole batch if two events in one request share a UUID,- // and a source export can contain duplicate event UUIDs. Track UUIDs across the whole- // transaction so duplicates are dropped before they reach (and poison) a capture request.+ // Capture's V1 endpoint keeps only the first event per UUID in one request, and a source+ // export can repeat a UUID across requests too. Track UUIDs across the whole transaction so+ // each UUID is sent to capture once.seen_uuids: Mutex<HashSet<Uuid>>,}@@ -99,7 +99,7 @@ impl<'a> Transaction<'a> for CaptureTransaction<'a> {let mut duplicates: u64 = 0;for captured in data {- // Drop repeat UUIDs so a source-level duplicate can't make capture reject the batch.+ // Drop repeat UUIDs so each one reaches capture once, even across requests.if seen.contains(&captured.inner.uuid) {duplicates += 1;continue;diff --git a/rust/capture/src/v1/analytics/process.rs b/rust/capture/src/v1/analytics/process.rsindex 73563b03ded4..44d70d5d736e 100644--- a/rust/capture/src/v1/analytics/process.rs+++ b/rust/capture/src/v1/analytics/process.rs@@ -31,6 +31,7 @@ use crate::config::CaptureMode;use crate::ingestion_warnings::{bounded_detail, emit_rate_limit_warning};use crate::router;use crate::sinks::sink::{Outcome, SinkResult};+use crate::v1::constants::CAPTURE_V1_WARNING_METRIC;use crate::v1::context::RequestContext;use crate::v1::prepare::{serialize_batch, SerializationFailure};use crate::v1::types::{Destination, Publishable};@@ -68,9 +69,63 @@ pub async fn process_batch(context: &mut Context,batch: Batch,) -> Result<BatchResponse, Error> {+ let (batch, duplicates_dropped) = drop_duplicate_uuids(batch);+ if !duplicates_dropped.is_empty() {+ observe_duplicate_drops(state, context, &duplicates_dropped);+ }let events = run_pipeline(state, context, batch).await?;emit_drop_warnings(state, context, &events);- Ok(BatchResponse::build(context, &events))+ Ok(BatchResponse::build(context, &events).with_duplicates_dropped(&duplicates_dropped))+}++/// Keeps the first event for each uuid and drops later copies, so one reused uuid+/// costs only its extra copies instead of rejecting the whole request.+fn drop_duplicate_uuids(mut batch: Batch) -> (Batch, HashMap<Uuid, u32>) {+ let mut seen: HashSet<Uuid> = HashSet::with_capacity(batch.batch.len());+ let mut dropped: HashMap<Uuid, u32> = HashMap::new();+ batch.batch.retain(|event| {+ // `validate_events` rejects an empty or unparseable uuid, so keep it for that check.+ let Ok(uuid) = Uuid::parse_str(&event.uuid) else {+ return true;+ };+ if seen.insert(uuid) {+ return true;+ }+ *dropped.entry(uuid).or_insert(0) += 1;+ false+ });+ (batch, dropped)+}++fn observe_duplicate_drops(state: &router::State, context: &Context, dropped: &HashMap<Uuid, u32>) {+ let count: u64 = dropped.values().map(|&n| u64::from(n)).sum();+ metrics::counter!(+ CAPTURE_V1_EVENTS_DROPPED,+ "reason" => "duplicate_event_uuid",+ "stage" => "deduplication",+ )+ .increment(count);+ metrics::counter!(+ CAPTURE_V1_WARNING_METRIC,+ "reason" => "duplicate_event_uuid",+ "path" => context.path,+ )+ .increment(1);+ crate::ctx_log!(+ Level::INFO,+ context,+ dropped_events = count,+ uuids = dropped.len(),+ "dropped later copies of duplicate event uuids"+ );+ emit_request_warning(+ state.ingestion_warning_emitter.as_deref(),+ &context.warning_context(),+ CAPTURE_V1_ANALYTICS,+ WarningType::DuplicateEventUuid,+ serde_json::Map::new(),+ count,+ );}/// Every stage of the v1 pipeline, from validation to publishing. Returns the@@ -480,7 +535,6 @@ fn validate_batch(batch: &Batch) -> Result<(), Error> {fn validate_events(context: &RequestContext, batch: Batch) -> Result<Vec<ValidatedEvent>, Error> {let batch_len = batch.batch.len();let mut events: Vec<ValidatedEvent> = Vec::with_capacity(batch_len);- let mut seen: HashSet<Uuid> = HashSet::with_capacity(batch_len);let mut illegal_distinct_id_count: u64 = 0;for event in batch.batch.into_iter() {@@ -490,12 +544,6 @@ fn validate_events(context: &RequestContext, batch: Batch) -> Result<Vec<Validatlet uuid = Uuid::parse_str(&event.uuid).map_err(|_| {count_validation_abort(Error::InvalidEventUuid(event.uuid.clone()), batch_len)})?;- if !seen.insert(uuid) {- return Err(count_validation_abort(- Error::DuplicateEventUuid(event.uuid.clone()),- batch_len,- ));- }match validate_event(&event) {Ok(raw_ts) => {@@ -1187,6 +1235,7 @@ mod tests {Pipeline, Restriction, RestrictionManager, RestrictionScope, RestrictionType,};use crate::v1::analytics::constants::{CAPTURE_V1_PATH, DETAIL_INVALID_OPTIONS};+ use crate::v1::analytics::response::BatchEntryStatus;use crate::v1::analytics::types::{Batch, Event};use crate::v1::prepare::DEFAULT_SCATTER_GATHER_MIN_BATCH;use crate::v1::test_utils::{@@ -1194,6 +1243,7 @@ mod tests {};use crate::v1::types::Destination;use crate::v1::Error;+ use rstest::rstest;fn valid_batch(events: Vec<Event>) -> Batch {Batch {@@ -1576,26 +1626,32 @@ mod tests {// --- validate_events ---#[test]- fn validate_events_duplicate_uuid_bails_batch() {- let ctx = test_utils::test_context();- let shared_uuid = Uuid::new_v4().to_string();- let batch = Batch {- created_at: "2026-03-19T14:30:00.000Z".to_string(),- historical_migration: false,- capture_internal: None,- batch: vec![- Event {- uuid: shared_uuid.clone(),- ..valid_event()- },- Event {- uuid: shared_uuid,- ..valid_event()- },- ],+ fn drop_duplicate_uuids_keeps_first_copy_and_counts_the_rest() {+ let shared = Uuid::new_v4();+ let other = Uuid::new_v4().to_string();+ let event = |uuid: String, distinct_id: &str| Event {+ uuid,+ distinct_id: distinct_id.to_string(),+ ..valid_event()};- let err = validate_events(&ctx, batch).unwrap_err();- assert!(matches!(err, Error::DuplicateEventUuid(_)));+ let batch = valid_batch(vec![+ event(shared.to_string(), "first"),+ event(other, "unique"),+ // Another spelling of the same uuid is still a duplicate.+ event(+ format!("{{{}}}", shared.to_string().to_uppercase()),+ "second",+ ),+ event(String::new(), "empty-uuid"),+ event(shared.to_string(), "third"),+ event("not-a-uuid".to_string(), "invalid-uuid"),+ ]);++ let (batch, dropped) = drop_duplicate_uuids(batch);++ let kept: Vec<&str> = batch.batch.iter().map(|e| e.distinct_id.as_str()).collect();+ assert_eq!(kept, vec!["first", "unique", "empty-uuid", "invalid-uuid"]);+ assert_eq!(dropped, HashMap::from([(shared, 2)]));}#[test]@@ -1636,33 +1692,22 @@ mod tests {// the bad one) — that is the per-event loss the single request-level// capture_v1_analytics_error tick can't show.- #[test]- fn duplicate_uuid_abort_counts_whole_batch() {- let ctx = test_utils::test_context();- let shared = Uuid::new_v4().to_string();- // dup is only detected at index 2, but all 3 events are lost.- let batch = valid_batch(vec![- Event {- uuid: shared.clone(),- ..valid_event()- },- Event {- uuid: Uuid::new_v4().to_string(),- ..valid_event()- },- Event {- uuid: shared,- ..valid_event()- },- ]);+ #[tokio::test]+ async fn duplicate_drops_count_each_dropped_copy() {+ let state = crate::v1::test_utils::TestStateBuilder::new().build().state;+ let ctx = test_utils::test_analytics_context();+ let dropped = HashMap::from([(Uuid::new_v4(), 2), (Uuid::new_v4(), 1)]);- let count = dropped_count("duplicate_event_uuid", "validation_abort", || {- assert!(matches!(- validate_events(&ctx, batch).unwrap_err(),- Error::DuplicateEventUuid(_)- ));+ let count = dropped_count("duplicate_event_uuid", "deduplication", || {+ observe_duplicate_drops(&state, &ctx, &dropped);});assert_eq!(count, Some(3));+ let requests = counter_value(+ CAPTURE_V1_WARNING_METRIC,+ &[("reason", "duplicate_event_uuid"), ("path", ctx.path)],+ || observe_duplicate_drops(&state, &ctx, &dropped),+ );+ assert_eq!(requests, Some(1), "one request-level count per request");}#[test]@@ -3199,14 +3244,50 @@ mod tests {);}- #[test]- fn validate_events_duplicate_uuid_via_realistic_pair() {- // Two distinct events (name / distinct_id / props) colliding on uuid.- let ctx = test_utils::test_context();- let (first, second) = test_utils::realistic_dup_uuid_pair();- let batch = valid_batch(vec![first, second]);- let err = validate_events(&ctx, batch).unwrap_err();- assert!(matches!(err, Error::DuplicateEventUuid(_)));+ #[rstest]+ #[case::analytics(CaptureMode::Events)]+ #[case::ai(CaptureMode::Ai)]+ #[tokio::test]+ async fn process_batch_keeps_first_duplicate_with_its_own_result(#[case] mode: CaptureMode) {+ let state = crate::v1::test_utils::TestStateBuilder::new()+ .with_capture_mode(mode)+ .build()+ .state;+ let mut ctx = test_utils::test_analytics_context();+ // The first copy is invalid, so its own drop must survive alongside the count.+ let (mut first, second) = test_utils::realistic_dup_uuid_pair();+ first.event = String::new();+ let shared = Uuid::parse_str(&first.uuid).unwrap();+ let third = Event {+ uuid: first.uuid.clone(),+ ..valid_event()+ };+ let unique = valid_event();+ let unique_uuid = Uuid::parse_str(&unique.uuid).unwrap();++ let resp = process_batch(+ &state,+ &mut ctx,+ valid_batch(vec![first, second, unique, third]),+ )+ .await+ .unwrap();++ let entries: HashMap<Uuid, &BatchEntryStatus> = resp+ .entries()+ .iter()+ .map(|(uuid, status)| (*uuid, status))+ .collect();+ assert_eq!(entries.len(), 2);+ assert_eq!(+ *entries[&shared],+ BatchEntryStatus {+ result: EventResult::Drop,+ details: Some("missing_event_name"),+ duplicates_dropped: 2,+ }+ );+ assert_eq!(entries[&unique_uuid].duplicates_dropped, 0);}#[tokio::test]@@ -4936,27 +5017,43 @@ mod tests {}#[tokio::test]- async fn warnings_batch_abort_emits_once_with_full_batch_count() {+ async fn warnings_duplicate_copies_emit_once_with_dropped_count() {let (state, collector) = state_with_warning_collector();let mut ctx = test_utils::test_analytics_context();- let shared_uuid = Uuid::new_v4().to_string();- let dup_a = Event {- uuid: shared_uuid.clone(),+ let first = valid_event();+ let copy = |distinct_id: &str| Event {+ uuid: first.uuid.clone(),+ distinct_id: distinct_id.to_string(),..valid_event()};- let dup_b = Event {- uuid: shared_uuid,- distinct_id: "someone-else".to_string(),+ let (copy_b, copy_c) = (copy("b"), copy("c"));+ let batch = valid_batch(vec![first, copy_b, valid_event(), copy_c]);++ let resp = process_batch(&state, &mut ctx, batch).await.unwrap();++ assert_eq!(resp.entries().len(), 2);+ let emitted = collector.emitted();+ assert_eq!(emitted.len(), 1);+ assert_eq!(emitted[0].warning, WarningType::DuplicateEventUuid);+ assert_eq!(emitted[0].count, 2, "only the dropped copies are charged");+ }++ #[tokio::test]+ async fn warnings_batch_abort_emits_once_with_full_batch_count() {+ let (state, collector) = state_with_warning_collector();+ let mut ctx = test_utils::test_analytics_context();+ let bad_uuid = Event {+ uuid: "not-a-uuid".to_string(),..valid_event()};- let batch = valid_batch(vec![dup_a, dup_b, valid_event()]);+ let batch = valid_batch(vec![valid_event(), bad_uuid, valid_event()]);let err = process_batch(&state, &mut ctx, batch).await.unwrap_err();- assert!(matches!(err, Error::DuplicateEventUuid(_)));+ assert!(matches!(err, Error::InvalidEventUuid(_)));let emitted = collector.emitted();assert_eq!(emitted.len(), 1);- assert_eq!(emitted[0].warning, WarningType::DuplicateEventUuid);+ assert_eq!(emitted[0].warning, WarningType::InvalidEventUuid);assert_eq!(emitted[0].count, 3,"a batch abort charges the whole batch length"diff --git a/rust/capture/src/v1/analytics/response.rs b/rust/capture/src/v1/analytics/response.rsindex a72bfd8a200a..51a906bb0942 100644--- a/rust/capture/src/v1/analytics/response.rs+++ b/rust/capture/src/v1/analytics/response.rs@@ -1,3 +1,5 @@+use std::collections::HashMap;+use axum::http::{header, HeaderValue, StatusCode};use axum::response::IntoResponse;use serde::ser::{SerializeMap, SerializeStruct};@@ -17,16 +19,22 @@ use crate::v1::context::RequestContext;pub struct BatchEntryStatus {pub result: EventResult,pub details: Option<&'static str>,+ /// Later events in the request with this uuid; capture dropped them and kept this one.+ pub duplicates_dropped: u32,}impl Serialize for BatchEntryStatus {fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {- let field_count = if self.details.is_some() { 2 } else { 1 };+ let field_count =+ 1 + usize::from(self.details.is_some()) + usize::from(self.duplicates_dropped > 0);let mut state = serializer.serialize_struct("BatchEntryStatus", field_count)?;state.serialize_field("result", &self.result)?;if let Some(detail) = self.details {state.serialize_field("details", detail)?;}+ if self.duplicates_dropped > 0 {+ state.serialize_field("duplicates_dropped", &self.duplicates_dropped)?;+ }state.end()}}@@ -61,6 +69,7 @@ impl BatchResponse {BatchEntryStatus {result: ev.result,details: ev.details,+ duplicates_dropped: 0,},)})@@ -74,6 +83,15 @@ impl BatchResponse {Self { has_retry, entries }}+ pub fn with_duplicates_dropped(mut self, dropped: &HashMap<Uuid, u32>) -> Self {+ for (uuid, status) in self.entries.iter_mut() {+ if let Some(count) = dropped.get(uuid) {+ status.duplicates_dropped = *count;+ }+ }+ self+ }+pub fn entries(&self) -> &[(Uuid, BatchEntryStatus)] {&self.entries}@@ -156,6 +174,7 @@ mod tests {use axum::body::to_bytes;use axum::response::IntoResponse;use chrono::Utc;+ use rstest::rstest;use serde_json::Value;use uuid::Uuid;@@ -189,50 +208,50 @@ mod tests {}}- #[test]- fn batch_entry_status_serialize_ok() {- let status = BatchEntryStatus {- result: EventResult::Ok,- details: None,- };- let json = serde_json::to_string(&status).unwrap();- assert_eq!(json, r#"{"result":"ok"}"#);- }-- #[test]- fn batch_entry_status_serialize_with_details() {+ #[rstest]+ #[case::ok(EventResult::Ok, None, 0, r#"{"result":"ok"}"#)]+ #[case::retry_with_details(+ EventResult::Retry,+ Some("not_persisted"),+ 0,+ r#"{"result":"retry","details":"not_persisted"}"#+ )]+ #[case::drop_with_details(+ EventResult::Drop,+ Some("billing_limit_exceeded"),+ 0,+ r#"{"result":"drop","details":"billing_limit_exceeded"}"#+ )]+ #[case::warning_with_details(+ EventResult::Warning,+ Some("person_processing_disabled"),+ 0,+ r#"{"result":"warning","details":"person_processing_disabled"}"#+ )]+ #[case::ok_with_duplicates(+ EventResult::Ok,+ None,+ 2,+ r#"{"result":"ok","duplicates_dropped":2}"#+ )]+ #[case::drop_with_details_and_duplicates(+ EventResult::Drop,+ Some("invalid_options"),+ 1,+ r#"{"result":"drop","details":"invalid_options","duplicates_dropped":1}"#+ )]+ fn batch_entry_status_serialize(+ #[case] result: EventResult,+ #[case] details: Option<&'static str>,+ #[case] duplicates_dropped: u32,+ #[case] expected: &str,+ ) {let status = BatchEntryStatus {- result: EventResult::Retry,- details: Some("not_persisted"),- };- let json = serde_json::to_string(&status).unwrap();- assert_eq!(json, r#"{"result":"retry","details":"not_persisted"}"#);- }-- #[test]- fn batch_entry_status_serialize_drop_with_details() {- let status = BatchEntryStatus {- result: EventResult::Drop,- details: Some("billing_limit_exceeded"),- };- let json = serde_json::to_string(&status).unwrap();- assert_eq!(- json,- r#"{"result":"drop","details":"billing_limit_exceeded"}"#- );- }-- #[test]- fn batch_entry_status_serialize_warning_with_details() {- let status = BatchEntryStatus {- result: EventResult::Warning,- details: Some("person_processing_disabled"),+ result,+ details,+ duplicates_dropped,};- let json = serde_json::to_string(&status).unwrap();- assert_eq!(- json,- r#"{"result":"warning","details":"person_processing_disabled"}"#- );+ assert_eq!(serde_json::to_string(&status).unwrap(), expected);}#[test]diff --git a/rust/capture/src/v1/error.rs b/rust/capture/src/v1/error.rsindex c8553da8a4a4..fc7448f45ad2 100644--- a/rust/capture/src/v1/error.rs+++ b/rust/capture/src/v1/error.rs@@ -122,8 +122,6 @@ pub enum Error {MissingEventUuid,#[error("event uuid is not valid: {0}")]InvalidEventUuid(String),- #[error("duplicate event uuid: {0}")]- DuplicateEventUuid(String),#[error("event submitted with invalid timestamp")]InvalidEventTimestamp,#[error("event properties is not a JSON object")]@@ -188,7 +186,6 @@ impl Error {Self::DistinctIdTooLarge => "distinct_id_too_large",Self::MissingEventUuid => "missing_event_uuid",Self::InvalidEventUuid(_) => "invalid_event_uuid",- Self::DuplicateEventUuid(_) => "duplicate_event_uuid",Self::InvalidEventTimestamp => "invalid_event_timestamp",Self::MalformedEventProperties => "malformed_event_properties",Self::DroppedPerformanceEvent => "dropped_performance_event",@@ -247,7 +244,6 @@ impl Error {| Self::DistinctIdTooLarge| Self::MissingEventUuid| Self::InvalidEventUuid(_)- | Self::DuplicateEventUuid(_)| Self::InvalidEventTimestamp| Self::MalformedEventProperties| Self::DroppedPerformanceEvent@@ -309,7 +305,6 @@ impl Error {| Self::DistinctIdTooLarge| Self::MissingEventUuid| Self::InvalidEventUuid(_)- | Self::DuplicateEventUuid(_)| Self::InvalidEventTimestamp| Self::MalformedEventProperties| Self::DroppedPerformanceEvent => StatusCode::BAD_REQUEST,diff --git a/rust/capture/tests/v1_pipeline.rs b/rust/capture/tests/v1_pipeline.rsindex 6e814e28d1fc..4964c24adda1 100644--- a/rust/capture/tests/v1_pipeline.rs+++ b/rust/capture/tests/v1_pipeline.rs@@ -247,23 +247,23 @@ async fn empty_batch_error() {}#[tokio::test]-async fn duplicate_uuid_error() {- let shared_uuid = Uuid::new_v4().to_string();- let mut e1 = valid_event();- e1.uuid = shared_uuid.clone();- let mut e2 = valid_event();- e2.event = "$identify".to_string();- e2.uuid = shared_uuid;+async fn duplicate_uuid_keeps_and_publishes_only_the_first_copy() {+ let first = event_with_name("$pageview");+ let mut copy = event_with_name("$exception");+ copy.uuid = first.uuid.clone();+ let payload = batch_payload(&[first, copy]);- let payload = batch_payload(&[e1, e2]);-- let err = run_batch(&payload, TestStateBuilder::new())- .await- .expect_err("duplicate uuid should fail");+ let (result, ts) = run_batch_with_state(&payload, TestStateBuilder::new()).await;- assert!(- matches!(err, Error::DuplicateEventUuid(_)),- "expected DuplicateEventUuid, got: {err:?}"+ let resp = result.expect("a duplicate uuid no longer rejects the request");+ assert_eq!(resp.entries().len(), 1);+ assert_eq!(resp.entries()[0].1.result, EventResult::Ok);+ assert_eq!(resp.entries()[0].1.duplicates_dropped, 1);+ let records = ts.mock_producer.get_records();+ assert_eq!(records.len(), 1, "only the first copy is published");+ assert_eq!(+ &*records[0].topic, "events_main",+ "the $exception copy was dropped");}