:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Overflow-reason stamping shared by the analytics, AI (`/i/v0/ai`) and OTEL
//! (`/i/v0/ai/otel`) handlers. The AI and OTEL handlers bypass
//! `events::analytics::process_events`, so each one calls
//! [`stamp_overflow_reason`] itself. The helper is sync because the
//! `OverflowLimiter` check is sync.
use std::sync::Arc;
use limiters::overflow::{OverflowLimiter, OverflowLimiterResult};
use metrics::counter;
use crate::v0_request::{DataType, OverflowReason, ProcessedEvent};
/// Stamp `ProcessedEventMetadata::overflow_reason` on every overflowing-lane
/// event in `events`, consulting the lane's limiter and the pre-existing
/// `force_overflow` flag stamped by event restrictions.
///
/// Two lanes can overflow: `AnalyticsMain` and `AiEvents`. Each lane consults
/// its own limiter instance — `analytics_limiter` for `AnalyticsMain`, `ai_limiter` for `AiEvents` — so per-key budgets are
/// isolated: analytics volume never pushes a key's AI events into AI overflow
/// and vice versa. The stamp carries no topic; the kafka sink maps it to the
/// lane's own overflow topic, so an overflowing AI event lands on AI
/// overflow, never analytics overflow.
///
/// Behavior:
/// * Events on non-overflowing lanes are skipped (heatmaps, exceptions,
/// client-ingestion-warnings, etc. never overflow).
/// * `force_overflow = true` (set upstream by event restrictions) emits the
/// `event_restriction` counter and short-circuits — the limiter is NOT
/// consulted. This runs on both
/// overflowing lanes regardless of limiter presence: the sink honors
/// `force_overflow` on its own (restriction rerouting must survive
/// `OVERFLOW_ENABLED=false`). The counter can over-report in one corner —
/// a forced AI event on Import is counted here but ignored by the sink;
/// only the sink knows the deployment's lanes.
/// * If the lane's limiter is `None`, the event passes through untouched.
/// * If the lane's limiter is `Some`, `is_limited(event.key())` is consulted
/// and the resulting [`OverflowReason`] is stamped, plus the matching
/// counter. `ForceLimited` additionally sets `skip_person_processing =
/// true` for pipeline-level readers of the flag; the sink derives the
/// `force_disable_person_processing` header from the reason itself, so
/// the header does not depend on this stamp.
///
/// Dashboards filter on the `reason` label of `capture_events_rerouted_overflow`,
/// so keep its values stable.
pub fn stamp_overflow_reason(
events: &mut [ProcessedEvent],
analytics_limiter: Option<&Arc<OverflowLimiter>>,
ai_limiter: Option<&Arc<OverflowLimiter>>,
) {
for event in events.iter_mut() {
let lane_limiter = match event.metadata.data_type {
DataType::AnalyticsMain => analytics_limiter,
DataType::AiEvents => ai_limiter,
_ => continue,
};
if event.metadata.force_overflow {
counter!(
"capture_events_rerouted_overflow",
"reason" => "event_restriction",
)
.increment(1);
continue;
}
let Some(limiter) = lane_limiter else {
continue;
};
let event_key = event.event.key();
match limiter.is_limited(&event_key) {
OverflowLimiterResult::ForceLimited => {
counter!(
"capture_events_rerouted_overflow",
"reason" => "force_limited",
)
.increment(1);
event.metadata.overflow_reason = Some(OverflowReason::ForceLimited);
// ForceLimited implies person processing is skipped; the sink
// derives that from the reason itself
// (`person_processing_disabled`). The flag is stamped
// alongside so pipeline-level readers of the flag see the
// same truth.
event.metadata.skip_person_processing = true;
}
OverflowLimiterResult::Limited => {
counter!(
"capture_events_rerouted_overflow",
"reason" => "rate_limited",
)
.increment(1);
event.metadata.overflow_reason = Some(OverflowReason::RateLimited {
preserve_locality: limiter.should_preserve_locality(),
});
}
OverflowLimiterResult::NotLimited => {}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::uuid_v7_from_datetime;
use crate::v0_request::{ProcessedEvent, ProcessedEventMetadata};
use common_types::CapturedEvent;
use std::num::NonZeroU32;
fn build_event(
data_type: DataType,
token: &str,
distinct_id: &str,
force_overflow: bool,
) -> ProcessedEvent {
let timestamp = chrono::Utc::now();
let event = CapturedEvent {
uuid: uuid_v7_from_datetime(timestamp),
distinct_id: distinct_id.to_string(),
session_id: None,
ip: "127.0.0.1".to_string(),
data: "{}".to_string(),
now: "2026-04-20T00:00:00Z".to_string(),
sent_at: None,
token: token.to_string(),
event: "test".to_string(),
timestamp,
is_cookieless_mode: false,
historical_migration: false,
};
let metadata = ProcessedEventMetadata {
data_type,
session_id: None,
computed_timestamp: None,
event_name: "test".to_string(),
force_overflow,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
skip_heatmap_processing: false,
overflow_reason: None,
distinct_id_truncated_from: None,
};
ProcessedEvent { event, metadata }
}
fn build_limiter(
per_second: u32,
burst: u32,
keys_to_reroute: Option<String>,
preserve_locality: bool,
) -> Arc<OverflowLimiter> {
Arc::new(OverflowLimiter::new(
NonZeroU32::new(per_second).unwrap(),
NonZeroU32::new(burst).unwrap(),
keys_to_reroute,
preserve_locality,
))
}
#[test]
fn force_overflow_short_circuits_without_consulting_limiter() {
// Use a limiter that would otherwise force-route this event; the
// short-circuit must skip the limiter call so the stamped reason
// stays None (force_overflow drives the sink directly without a
// RateLimited/ForceLimited stamp).
let limiter = build_limiter(10, 10, Some("phc_t:user".to_string()), false);
let mut events = vec![build_event(
DataType::AnalyticsMain,
"phc_t",
"user",
true, // force_overflow
)];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(
events[0].metadata.overflow_reason, None,
"force_overflow short-circuits before the limiter; reason stays None"
);
assert!(
!events[0].metadata.skip_person_processing,
"force_overflow alone must not flip skip_person_processing"
);
}
#[test]
fn force_limited_stamps_reason_and_skip_person_processing() {
let limiter = build_limiter(10, 10, Some("phc_t:user".to_string()), false);
let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "user", false)];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(
events[0].metadata.overflow_reason,
Some(OverflowReason::ForceLimited)
);
assert!(
events[0].metadata.skip_person_processing,
"ForceLimited must set skip_person_processing alongside the reason"
);
}
#[test]
fn rate_limited_stamps_preserve_locality_true() {
// burst=1 means the second event exceeds the budget; preserve_locality
// is mirrored from the limiter config onto the stamped reason.
let limiter = build_limiter(1, 1, None, true);
let mut events = vec![
build_event(DataType::AnalyticsMain, "phc_t", "u", false),
build_event(DataType::AnalyticsMain, "phc_t", "u", false),
];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(
events[0].metadata.overflow_reason, None,
"first event within burst must not be stamped"
);
assert_eq!(
events[1].metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: true
})
);
assert!(
!events[1].metadata.skip_person_processing,
"RateLimited (non-Force) must not flip skip_person_processing"
);
}
#[test]
fn rate_limited_stamps_preserve_locality_false() {
let limiter = build_limiter(1, 1, None, false);
let mut events = vec![
build_event(DataType::AnalyticsMain, "phc_t", "u", false),
build_event(DataType::AnalyticsMain, "phc_t", "u", false),
];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(
events[1].metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: false
})
);
}
#[test]
fn not_limited_leaves_reason_none() {
// burst=10, single event; well under the budget.
let limiter = build_limiter(10, 10, None, false);
let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", false)];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(events[0].metadata.overflow_reason, None);
}
#[test]
fn non_analytics_main_events_are_skipped() {
let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
let mut events = vec![
build_event(DataType::SnapshotMain, "phc_t", "u", false),
build_event(DataType::HeatmapMain, "phc_t", "u", false),
build_event(DataType::ExceptionErrorTracking, "phc_t", "u", false),
build_event(DataType::ClientIngestionWarning, "phc_t", "u", false),
build_event(DataType::AnalyticsHistorical, "phc_t", "u", false),
build_event(DataType::AiEvents, "phc_t", "u", false),
];
stamp_overflow_reason(&mut events, Some(&limiter), None);
for (i, ev) in events.iter().enumerate() {
assert_eq!(
ev.metadata.overflow_reason, None,
"event[{i}] data_type {:?} must be skipped",
ev.metadata.data_type
);
assert!(!ev.metadata.skip_person_processing, "event[{i}]: untouched");
}
}
#[test]
fn none_limiter_is_a_no_op_for_non_force_overflow_events() {
let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", false)];
stamp_overflow_reason(&mut events, None, None);
assert_eq!(events[0].metadata.overflow_reason, None);
assert!(!events[0].metadata.skip_person_processing);
}
#[test]
fn none_limiter_still_emits_event_restriction_counter() {
// force_overflow is independent of the limiter, so the
// event_restriction short-circuit must still fire (and emit its
// counter) even when the limiter is absent.
let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", true)];
stamp_overflow_reason(&mut events, None, None);
assert_eq!(
events[0].metadata.overflow_reason, None,
"force_overflow leaves overflow_reason as None"
);
}
#[test]
fn empty_batch_is_a_no_op() {
let limiter = build_limiter(10, 10, None, false);
let mut events: Vec<ProcessedEvent> = Vec::new();
stamp_overflow_reason(&mut events, Some(&limiter), None);
}
#[test]
fn mixed_batch_stamps_only_analytics_main_entries() {
// Realistic OTEL-shaped batch: a few AnalyticsMain spans plus one
// HeatmapMain that snuck in. Only the AnalyticsMain over-budget
// entries should be stamped.
let limiter = build_limiter(1, 1, None, true);
let mut events = vec![
build_event(DataType::AnalyticsMain, "phc_t", "user_a", false),
build_event(DataType::HeatmapMain, "phc_t", "user_a", false),
build_event(DataType::AnalyticsMain, "phc_t", "user_a", false),
];
stamp_overflow_reason(&mut events, Some(&limiter), None);
assert_eq!(
events[0].metadata.overflow_reason, None,
"first AnalyticsMain within burst"
);
assert_eq!(
events[1].metadata.overflow_reason, None,
"HeatmapMain must be skipped regardless of limiter state"
);
assert_eq!(
events[2].metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: true
}),
"second AnalyticsMain over budget"
);
}
#[rstest::rstest]
#[case::limiter_present(true, Some(OverflowReason::ForceLimited), true)]
#[case::limiter_absent(false, None, false)]
fn ai_events_stamping_gated_on_limiter_presence(
#[case] ai_limiter_present: bool,
#[case] expected_reason: Option<OverflowReason>,
#[case] expected_skip_person: bool,
) {
let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
let ai_limiter = ai_limiter_present.then_some(&limiter);
let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", false)];
stamp_overflow_reason(&mut events, None, ai_limiter);
assert_eq!(events[0].metadata.overflow_reason, expected_reason);
assert_eq!(
events[0].metadata.skip_person_processing,
expected_skip_person
);
}
#[test]
fn ai_events_rate_limited_mirrors_preserve_locality() {
// The AI limiter's preserve-partition-locality flag is mirrored onto
// the stamped reason, exactly like the analytics lane.
let limiter = build_limiter(1, 1, None, true);
let mut events = vec![
build_event(DataType::AiEvents, "phc_t", "u", false),
build_event(DataType::AiEvents, "phc_t", "u", false),
];
stamp_overflow_reason(&mut events, None, Some(&limiter));
assert_eq!(events[0].metadata.overflow_reason, None);
assert_eq!(
events[1].metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: true
})
);
}
/// The two lanes consult separate limiter instances, so one lane
/// exhausting a key's budget must not stamp the other lane's events for
/// the same `token:distinct_id` key.
#[rstest::rstest]
#[case::analytics_burst_leaves_ai_untouched(DataType::AnalyticsMain, DataType::AiEvents)]
#[case::ai_burst_leaves_analytics_untouched(DataType::AiEvents, DataType::AnalyticsMain)]
fn lane_budgets_are_isolated_per_key(
#[case] bursting_lane: DataType,
#[case] quiet_lane: DataType,
) {
// burst=1 for both limiters: the bursting lane's second event goes
// over budget, while the quiet lane's single event (same key) must
// stay within its own untouched budget.
let analytics_limiter = build_limiter(1, 1, None, false);
let ai_limiter = build_limiter(1, 1, None, false);
let mut events = vec![
build_event(bursting_lane, "phc_t", "u", false),
build_event(bursting_lane, "phc_t", "u", false),
build_event(quiet_lane, "phc_t", "u", false),
];
stamp_overflow_reason(&mut events, Some(&analytics_limiter), Some(&ai_limiter));
assert_eq!(events[0].metadata.overflow_reason, None);
assert_eq!(
events[1].metadata.overflow_reason,
Some(OverflowReason::RateLimited {
preserve_locality: false
}),
"second {bursting_lane:?} event over budget"
);
assert_eq!(
events[2].metadata.overflow_reason, None,
"{quiet_lane:?} must not inherit the {bursting_lane:?} lane's exhausted budget"
);
}
#[test]
fn ai_lane_ignores_analytics_limiter_when_own_limiter_absent() {
// A missing AI limiter means no rate-limit overflow for the AI lane,
// even when the analytics limiter would force-route the same key.
let analytics_limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", false)];
stamp_overflow_reason(&mut events, Some(&analytics_limiter), None);
assert_eq!(events[0].metadata.overflow_reason, None);
}
#[test]
fn ai_events_force_overflow_short_circuits() {
// Identical to the analytics lane: a restriction-driven
// force_overflow bypasses the limiter and leaves the reason unset;
// the sink routes on the flag alone.
let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", true)];
stamp_overflow_reason(&mut events, None, Some(&limiter));
assert_eq!(events[0].metadata.overflow_reason, None);
assert!(!events[0].metadata.skip_person_processing);
}
}