Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(capture): keep the first copy of a duplicate v1 event uuid (#122305)

Eli Reisman55c57e6ff550
diff --git a/rust/batch-import-worker/src/emit/capture.rs b/rust/batch-import-worker/src/emit/capture.rs
index 909a739c2be4..0ecbe0cf1731 100644
--- a/rust/batch-import-worker/src/emit/capture.rs
+++ b/rust/batch-import-worker/src/emit/capture.rs
@@ -25,9 +25,9 @@ pub struct CaptureTransaction<'a> {
send_rate: u64,
start: Instant,
events: Mutex<Vec<Event>>,
- // Capture's V1 endpoint rejects a whole batch if two events in one request share a UUID,
- // and a source export can contain duplicate event UUIDs. Track UUIDs across the whole
- // transaction so duplicates are dropped before they reach (and poison) a capture request.
+ // Capture's V1 endpoint keeps only the first event per UUID in one request, and a source
+ // export can repeat a UUID across requests too. Track UUIDs across the whole transaction so
+ // each UUID is sent to capture once.
seen_uuids: Mutex<HashSet<Uuid>>,
}
@@ -99,7 +99,7 @@ impl<'a> Transaction<'a> for CaptureTransaction<'a> {
let mut duplicates: u64 = 0;
for captured in data {
- // Drop repeat UUIDs so a source-level duplicate can't make capture reject the batch.
+ // Drop repeat UUIDs so each one reaches capture once, even across requests.
if seen.contains(&captured.inner.uuid) {
duplicates += 1;
continue;
diff --git a/rust/capture/src/v1/analytics/process.rs b/rust/capture/src/v1/analytics/process.rs
index 73563b03ded4..44d70d5d736e 100644
--- a/rust/capture/src/v1/analytics/process.rs
+++ b/rust/capture/src/v1/analytics/process.rs
@@ -31,6 +31,7 @@ use crate::config::CaptureMode;
use crate::ingestion_warnings::{bounded_detail, emit_rate_limit_warning};
use crate::router;
use crate::sinks::sink::{Outcome, SinkResult};
+use crate::v1::constants::CAPTURE_V1_WARNING_METRIC;
use crate::v1::context::RequestContext;
use crate::v1::prepare::{serialize_batch, SerializationFailure};
use crate::v1::types::{Destination, Publishable};
@@ -68,9 +69,63 @@ pub async fn process_batch(
context: &mut Context,
batch: Batch,
) -> Result<BatchResponse, Error> {
+ let (batch, duplicates_dropped) = drop_duplicate_uuids(batch);
+ if !duplicates_dropped.is_empty() {
+ observe_duplicate_drops(state, context, &duplicates_dropped);
+ }
let events = run_pipeline(state, context, batch).await?;
emit_drop_warnings(state, context, &events);
- Ok(BatchResponse::build(context, &events))
+ Ok(BatchResponse::build(context, &events).with_duplicates_dropped(&duplicates_dropped))
+}
+
+/// Keeps the first event for each uuid and drops later copies, so one reused uuid
+/// costs only its extra copies instead of rejecting the whole request.
+fn drop_duplicate_uuids(mut batch: Batch) -> (Batch, HashMap<Uuid, u32>) {
+ let mut seen: HashSet<Uuid> = HashSet::with_capacity(batch.batch.len());
+ let mut dropped: HashMap<Uuid, u32> = HashMap::new();
+ batch.batch.retain(|event| {
+ // `validate_events` rejects an empty or unparseable uuid, so keep it for that check.
+ let Ok(uuid) = Uuid::parse_str(&event.uuid) else {
+ return true;
+ };
+ if seen.insert(uuid) {
+ return true;
+ }
+ *dropped.entry(uuid).or_insert(0) += 1;
+ false
+ });
+ (batch, dropped)
+}
+
+fn observe_duplicate_drops(state: &router::State, context: &Context, dropped: &HashMap<Uuid, u32>) {
+ let count: u64 = dropped.values().map(|&n| u64::from(n)).sum();
+ metrics::counter!(
+ CAPTURE_V1_EVENTS_DROPPED,
+ "reason" => "duplicate_event_uuid",
+ "stage" => "deduplication",
+ )
+ .increment(count);
+ metrics::counter!(
+ CAPTURE_V1_WARNING_METRIC,
+ "reason" => "duplicate_event_uuid",
+ "path" => context.path,
+ )
+ .increment(1);
+ crate::ctx_log!(
+ Level::INFO,
+ context,
+ dropped_events = count,
+ uuids = dropped.len(),
+ "dropped later copies of duplicate event uuids"
+ );
+ emit_request_warning(
+ state.ingestion_warning_emitter.as_deref(),
+ &context.warning_context(),
+ CAPTURE_V1_ANALYTICS,
+ WarningType::DuplicateEventUuid,
+ serde_json::Map::new(),
+ count,
+ );
}
/// Every stage of the v1 pipeline, from validation to publishing. Returns the
@@ -480,7 +535,6 @@ fn validate_batch(batch: &Batch) -> Result<(), Error> {
fn validate_events(context: &RequestContext, batch: Batch) -> Result<Vec<ValidatedEvent>, Error> {
let batch_len = batch.batch.len();
let mut events: Vec<ValidatedEvent> = Vec::with_capacity(batch_len);
- let mut seen: HashSet<Uuid> = HashSet::with_capacity(batch_len);
let mut illegal_distinct_id_count: u64 = 0;
for event in batch.batch.into_iter() {
@@ -490,12 +544,6 @@ fn validate_events(context: &RequestContext, batch: Batch) -> Result<Vec<Validat
let uuid = Uuid::parse_str(&event.uuid).map_err(|_| {
count_validation_abort(Error::InvalidEventUuid(event.uuid.clone()), batch_len)
})?;
- if !seen.insert(uuid) {
- return Err(count_validation_abort(
- Error::DuplicateEventUuid(event.uuid.clone()),
- batch_len,
- ));
- }
match validate_event(&event) {
Ok(raw_ts) => {
@@ -1187,6 +1235,7 @@ mod tests {
Pipeline, Restriction, RestrictionManager, RestrictionScope, RestrictionType,
};
use crate::v1::analytics::constants::{CAPTURE_V1_PATH, DETAIL_INVALID_OPTIONS};
+ use crate::v1::analytics::response::BatchEntryStatus;
use crate::v1::analytics::types::{Batch, Event};
use crate::v1::prepare::DEFAULT_SCATTER_GATHER_MIN_BATCH;
use crate::v1::test_utils::{
@@ -1194,6 +1243,7 @@ mod tests {
};
use crate::v1::types::Destination;
use crate::v1::Error;
+ use rstest::rstest;
fn valid_batch(events: Vec<Event>) -> Batch {
Batch {
@@ -1576,26 +1626,32 @@ mod tests {
// --- validate_events ---
#[test]
- fn validate_events_duplicate_uuid_bails_batch() {
- let ctx = test_utils::test_context();
- let shared_uuid = Uuid::new_v4().to_string();
- let batch = Batch {
- created_at: "2026-03-19T14:30:00.000Z".to_string(),
- historical_migration: false,
- capture_internal: None,
- batch: vec![
- Event {
- uuid: shared_uuid.clone(),
- ..valid_event()
- },
- Event {
- uuid: shared_uuid,
- ..valid_event()
- },
- ],
+ fn drop_duplicate_uuids_keeps_first_copy_and_counts_the_rest() {
+ let shared = Uuid::new_v4();
+ let other = Uuid::new_v4().to_string();
+ let event = |uuid: String, distinct_id: &str| Event {
+ uuid,
+ distinct_id: distinct_id.to_string(),
+ ..valid_event()
};
- let err = validate_events(&ctx, batch).unwrap_err();
- assert!(matches!(err, Error::DuplicateEventUuid(_)));
+ let batch = valid_batch(vec![
+ event(shared.to_string(), "first"),
+ event(other, "unique"),
+ // Another spelling of the same uuid is still a duplicate.
+ event(
+ format!("{{{}}}", shared.to_string().to_uppercase()),
+ "second",
+ ),
+ event(String::new(), "empty-uuid"),
+ event(shared.to_string(), "third"),
+ event("not-a-uuid".to_string(), "invalid-uuid"),
+ ]);
+
+ let (batch, dropped) = drop_duplicate_uuids(batch);
+
+ let kept: Vec<&str> = batch.batch.iter().map(|e| e.distinct_id.as_str()).collect();
+ assert_eq!(kept, vec!["first", "unique", "empty-uuid", "invalid-uuid"]);
+ assert_eq!(dropped, HashMap::from([(shared, 2)]));
}
#[test]
@@ -1636,33 +1692,22 @@ mod tests {
// the bad one) — that is the per-event loss the single request-level
// capture_v1_analytics_error tick can't show.
- #[test]
- fn duplicate_uuid_abort_counts_whole_batch() {
- let ctx = test_utils::test_context();
- let shared = Uuid::new_v4().to_string();
- // dup is only detected at index 2, but all 3 events are lost.
- let batch = valid_batch(vec![
- Event {
- uuid: shared.clone(),
- ..valid_event()
- },
- Event {
- uuid: Uuid::new_v4().to_string(),
- ..valid_event()
- },
- Event {
- uuid: shared,
- ..valid_event()
- },
- ]);
+ #[tokio::test]
+ async fn duplicate_drops_count_each_dropped_copy() {
+ let state = crate::v1::test_utils::TestStateBuilder::new().build().state;
+ let ctx = test_utils::test_analytics_context();
+ let dropped = HashMap::from([(Uuid::new_v4(), 2), (Uuid::new_v4(), 1)]);
- let count = dropped_count("duplicate_event_uuid", "validation_abort", || {
- assert!(matches!(
- validate_events(&ctx, batch).unwrap_err(),
- Error::DuplicateEventUuid(_)
- ));
+ let count = dropped_count("duplicate_event_uuid", "deduplication", || {
+ observe_duplicate_drops(&state, &ctx, &dropped);
});
assert_eq!(count, Some(3));
+ let requests = counter_value(
+ CAPTURE_V1_WARNING_METRIC,
+ &[("reason", "duplicate_event_uuid"), ("path", ctx.path)],
+ || observe_duplicate_drops(&state, &ctx, &dropped),
+ );
+ assert_eq!(requests, Some(1), "one request-level count per request");
}
#[test]
@@ -3199,14 +3244,50 @@ mod tests {
);
}
- #[test]
- fn validate_events_duplicate_uuid_via_realistic_pair() {
- // Two distinct events (name / distinct_id / props) colliding on uuid.
- let ctx = test_utils::test_context();
- let (first, second) = test_utils::realistic_dup_uuid_pair();
- let batch = valid_batch(vec![first, second]);
- let err = validate_events(&ctx, batch).unwrap_err();
- assert!(matches!(err, Error::DuplicateEventUuid(_)));
+ #[rstest]
+ #[case::analytics(CaptureMode::Events)]
+ #[case::ai(CaptureMode::Ai)]
+ #[tokio::test]
+ async fn process_batch_keeps_first_duplicate_with_its_own_result(#[case] mode: CaptureMode) {
+ let state = crate::v1::test_utils::TestStateBuilder::new()
+ .with_capture_mode(mode)
+ .build()
+ .state;
+ let mut ctx = test_utils::test_analytics_context();
+ // The first copy is invalid, so its own drop must survive alongside the count.
+ let (mut first, second) = test_utils::realistic_dup_uuid_pair();
+ first.event = String::new();
+ let shared = Uuid::parse_str(&first.uuid).unwrap();
+ let third = Event {
+ uuid: first.uuid.clone(),
+ ..valid_event()
+ };
+ let unique = valid_event();
+ let unique_uuid = Uuid::parse_str(&unique.uuid).unwrap();
+
+ let resp = process_batch(
+ &state,
+ &mut ctx,
+ valid_batch(vec![first, second, unique, third]),
+ )
+ .await
+ .unwrap();
+
+ let entries: HashMap<Uuid, &BatchEntryStatus> = resp
+ .entries()
+ .iter()
+ .map(|(uuid, status)| (*uuid, status))
+ .collect();
+ assert_eq!(entries.len(), 2);
+ assert_eq!(
+ *entries[&shared],
+ BatchEntryStatus {
+ result: EventResult::Drop,
+ details: Some("missing_event_name"),
+ duplicates_dropped: 2,
+ }
+ );
+ assert_eq!(entries[&unique_uuid].duplicates_dropped, 0);
}
#[tokio::test]
@@ -4936,27 +5017,43 @@ mod tests {
}
#[tokio::test]
- async fn warnings_batch_abort_emits_once_with_full_batch_count() {
+ async fn warnings_duplicate_copies_emit_once_with_dropped_count() {
let (state, collector) = state_with_warning_collector();
let mut ctx = test_utils::test_analytics_context();
- let shared_uuid = Uuid::new_v4().to_string();
- let dup_a = Event {
- uuid: shared_uuid.clone(),
+ let first = valid_event();
+ let copy = |distinct_id: &str| Event {
+ uuid: first.uuid.clone(),
+ distinct_id: distinct_id.to_string(),
..valid_event()
};
- let dup_b = Event {
- uuid: shared_uuid,
- distinct_id: "someone-else".to_string(),
+ let (copy_b, copy_c) = (copy("b"), copy("c"));
+ let batch = valid_batch(vec![first, copy_b, valid_event(), copy_c]);
+
+ let resp = process_batch(&state, &mut ctx, batch).await.unwrap();
+
+ assert_eq!(resp.entries().len(), 2);
+ let emitted = collector.emitted();
+ assert_eq!(emitted.len(), 1);
+ assert_eq!(emitted[0].warning, WarningType::DuplicateEventUuid);
+ assert_eq!(emitted[0].count, 2, "only the dropped copies are charged");
+ }
+
+ #[tokio::test]
+ async fn warnings_batch_abort_emits_once_with_full_batch_count() {
+ let (state, collector) = state_with_warning_collector();
+ let mut ctx = test_utils::test_analytics_context();
+ let bad_uuid = Event {
+ uuid: "not-a-uuid".to_string(),
..valid_event()
};
- let batch = valid_batch(vec![dup_a, dup_b, valid_event()]);
+ let batch = valid_batch(vec![valid_event(), bad_uuid, valid_event()]);
let err = process_batch(&state, &mut ctx, batch).await.unwrap_err();
- assert!(matches!(err, Error::DuplicateEventUuid(_)));
+ assert!(matches!(err, Error::InvalidEventUuid(_)));
let emitted = collector.emitted();
assert_eq!(emitted.len(), 1);
- assert_eq!(emitted[0].warning, WarningType::DuplicateEventUuid);
+ assert_eq!(emitted[0].warning, WarningType::InvalidEventUuid);
assert_eq!(
emitted[0].count, 3,
"a batch abort charges the whole batch length"
diff --git a/rust/capture/src/v1/analytics/response.rs b/rust/capture/src/v1/analytics/response.rs
index a72bfd8a200a..51a906bb0942 100644
--- a/rust/capture/src/v1/analytics/response.rs
+++ b/rust/capture/src/v1/analytics/response.rs
@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
use axum::http::{header, HeaderValue, StatusCode};
use axum::response::IntoResponse;
use serde::ser::{SerializeMap, SerializeStruct};
@@ -17,16 +19,22 @@ use crate::v1::context::RequestContext;
pub struct BatchEntryStatus {
pub result: EventResult,
pub details: Option<&'static str>,
+ /// Later events in the request with this uuid; capture dropped them and kept this one.
+ pub duplicates_dropped: u32,
}
impl Serialize for BatchEntryStatus {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
- let field_count = if self.details.is_some() { 2 } else { 1 };
+ let field_count =
+ 1 + usize::from(self.details.is_some()) + usize::from(self.duplicates_dropped > 0);
let mut state = serializer.serialize_struct("BatchEntryStatus", field_count)?;
state.serialize_field("result", &self.result)?;
if let Some(detail) = self.details {
state.serialize_field("details", detail)?;
}
+ if self.duplicates_dropped > 0 {
+ state.serialize_field("duplicates_dropped", &self.duplicates_dropped)?;
+ }
state.end()
}
}
@@ -61,6 +69,7 @@ impl BatchResponse {
BatchEntryStatus {
result: ev.result,
details: ev.details,
+ duplicates_dropped: 0,
},
)
})
@@ -74,6 +83,15 @@ impl BatchResponse {
Self { has_retry, entries }
}
+ pub fn with_duplicates_dropped(mut self, dropped: &HashMap<Uuid, u32>) -> Self {
+ for (uuid, status) in self.entries.iter_mut() {
+ if let Some(count) = dropped.get(uuid) {
+ status.duplicates_dropped = *count;
+ }
+ }
+ self
+ }
+
pub fn entries(&self) -> &[(Uuid, BatchEntryStatus)] {
&self.entries
}
@@ -156,6 +174,7 @@ mod tests {
use axum::body::to_bytes;
use axum::response::IntoResponse;
use chrono::Utc;
+ use rstest::rstest;
use serde_json::Value;
use uuid::Uuid;
@@ -189,50 +208,50 @@ mod tests {
}
}
- #[test]
- fn batch_entry_status_serialize_ok() {
- let status = BatchEntryStatus {
- result: EventResult::Ok,
- details: None,
- };
- let json = serde_json::to_string(&status).unwrap();
- assert_eq!(json, r#"{"result":"ok"}"#);
- }
-
- #[test]
- fn batch_entry_status_serialize_with_details() {
+ #[rstest]
+ #[case::ok(EventResult::Ok, None, 0, r#"{"result":"ok"}"#)]
+ #[case::retry_with_details(
+ EventResult::Retry,
+ Some("not_persisted"),
+ 0,
+ r#"{"result":"retry","details":"not_persisted"}"#
+ )]
+ #[case::drop_with_details(
+ EventResult::Drop,
+ Some("billing_limit_exceeded"),
+ 0,
+ r#"{"result":"drop","details":"billing_limit_exceeded"}"#
+ )]
+ #[case::warning_with_details(
+ EventResult::Warning,
+ Some("person_processing_disabled"),
+ 0,
+ r#"{"result":"warning","details":"person_processing_disabled"}"#
+ )]
+ #[case::ok_with_duplicates(
+ EventResult::Ok,
+ None,
+ 2,
+ r#"{"result":"ok","duplicates_dropped":2}"#
+ )]
+ #[case::drop_with_details_and_duplicates(
+ EventResult::Drop,
+ Some("invalid_options"),
+ 1,
+ r#"{"result":"drop","details":"invalid_options","duplicates_dropped":1}"#
+ )]
+ fn batch_entry_status_serialize(
+ #[case] result: EventResult,
+ #[case] details: Option<&'static str>,
+ #[case] duplicates_dropped: u32,
+ #[case] expected: &str,
+ ) {
let status = BatchEntryStatus {
- result: EventResult::Retry,
- details: Some("not_persisted"),
- };
- let json = serde_json::to_string(&status).unwrap();
- assert_eq!(json, r#"{"result":"retry","details":"not_persisted"}"#);
- }
-
- #[test]
- fn batch_entry_status_serialize_drop_with_details() {
- let status = BatchEntryStatus {
- result: EventResult::Drop,
- details: Some("billing_limit_exceeded"),
- };
- let json = serde_json::to_string(&status).unwrap();
- assert_eq!(
- json,
- r#"{"result":"drop","details":"billing_limit_exceeded"}"#
- );
- }
-
- #[test]
- fn batch_entry_status_serialize_warning_with_details() {
- let status = BatchEntryStatus {
- result: EventResult::Warning,
- details: Some("person_processing_disabled"),
+ result,
+ details,
+ duplicates_dropped,
};
- let json = serde_json::to_string(&status).unwrap();
- assert_eq!(
- json,
- r#"{"result":"warning","details":"person_processing_disabled"}"#
- );
+ assert_eq!(serde_json::to_string(&status).unwrap(), expected);
}
#[test]
diff --git a/rust/capture/src/v1/error.rs b/rust/capture/src/v1/error.rs
index c8553da8a4a4..fc7448f45ad2 100644
--- a/rust/capture/src/v1/error.rs
+++ b/rust/capture/src/v1/error.rs
@@ -122,8 +122,6 @@ pub enum Error {
MissingEventUuid,
#[error("event uuid is not valid: {0}")]
InvalidEventUuid(String),
- #[error("duplicate event uuid: {0}")]
- DuplicateEventUuid(String),
#[error("event submitted with invalid timestamp")]
InvalidEventTimestamp,
#[error("event properties is not a JSON object")]
@@ -188,7 +186,6 @@ impl Error {
Self::DistinctIdTooLarge => "distinct_id_too_large",
Self::MissingEventUuid => "missing_event_uuid",
Self::InvalidEventUuid(_) => "invalid_event_uuid",
- Self::DuplicateEventUuid(_) => "duplicate_event_uuid",
Self::InvalidEventTimestamp => "invalid_event_timestamp",
Self::MalformedEventProperties => "malformed_event_properties",
Self::DroppedPerformanceEvent => "dropped_performance_event",
@@ -247,7 +244,6 @@ impl Error {
| Self::DistinctIdTooLarge
| Self::MissingEventUuid
| Self::InvalidEventUuid(_)
- | Self::DuplicateEventUuid(_)
| Self::InvalidEventTimestamp
| Self::MalformedEventProperties
| Self::DroppedPerformanceEvent
@@ -309,7 +305,6 @@ impl Error {
| Self::DistinctIdTooLarge
| Self::MissingEventUuid
| Self::InvalidEventUuid(_)
- | Self::DuplicateEventUuid(_)
| Self::InvalidEventTimestamp
| Self::MalformedEventProperties
| Self::DroppedPerformanceEvent => StatusCode::BAD_REQUEST,
diff --git a/rust/capture/tests/v1_pipeline.rs b/rust/capture/tests/v1_pipeline.rs
index 6e814e28d1fc..4964c24adda1 100644
--- a/rust/capture/tests/v1_pipeline.rs
+++ b/rust/capture/tests/v1_pipeline.rs
@@ -247,23 +247,23 @@ async fn empty_batch_error() {
}
#[tokio::test]
-async fn duplicate_uuid_error() {
- let shared_uuid = Uuid::new_v4().to_string();
- let mut e1 = valid_event();
- e1.uuid = shared_uuid.clone();
- let mut e2 = valid_event();
- e2.event = "$identify".to_string();
- e2.uuid = shared_uuid;
+async fn duplicate_uuid_keeps_and_publishes_only_the_first_copy() {
+ let first = event_with_name("$pageview");
+ let mut copy = event_with_name("$exception");
+ copy.uuid = first.uuid.clone();
+ let payload = batch_payload(&[first, copy]);
- let payload = batch_payload(&[e1, e2]);
-
- let err = run_batch(&payload, TestStateBuilder::new())
- .await
- .expect_err("duplicate uuid should fail");
+ let (result, ts) = run_batch_with_state(&payload, TestStateBuilder::new()).await;
- assert!(
- matches!(err, Error::DuplicateEventUuid(_)),
- "expected DuplicateEventUuid, got: {err:?}"
+ let resp = result.expect("a duplicate uuid no longer rejects the request");
+ assert_eq!(resp.entries().len(), 1);
+ assert_eq!(resp.entries()[0].1.result, EventResult::Ok);
+ assert_eq!(resp.entries()[0].1.duplicates_dropped, 1);
+ let records = ts.mock_producer.get_records();
+ assert_eq!(records.len(), 1, "only the first copy is published");
+ assert_eq!(
+ &*records[0].topic, "events_main",
+ "the $exception copy was dropped"
);
}