Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.3 KB
Raw Blame
//! Overflow-reason stamping shared by the analytics, AI (`/i/v0/ai`) and OTEL
//! (`/i/v0/ai/otel`) handlers. The AI and OTEL handlers bypass
//! `events::analytics::process_events`, so each one calls
//! [`stamp_overflow_reason`] itself. The helper is sync because the
//! `OverflowLimiter` check is sync.

use std::sync::Arc;

use limiters::overflow::{OverflowLimiter, OverflowLimiterResult};
use metrics::counter;

use crate::v0_request::{DataType, OverflowReason, ProcessedEvent};

/// Stamp `ProcessedEventMetadata::overflow_reason` on every overflowing-lane
/// event in `events`, consulting the lane's limiter and the pre-existing
/// `force_overflow` flag stamped by event restrictions.
///
/// Two lanes can overflow: `AnalyticsMain` and `AiEvents`. Each lane consults
/// its own limiter instance — `analytics_limiter` for `AnalyticsMain`, `ai_limiter` for `AiEvents` — so per-key budgets are
/// isolated: analytics volume never pushes a key's AI events into AI overflow
/// and vice versa. The stamp carries no topic; the kafka sink maps it to the
/// lane's own overflow topic, so an overflowing AI event lands on AI
/// overflow, never analytics overflow.
///
/// Behavior:
/// * Events on non-overflowing lanes are skipped (heatmaps, exceptions,
///   client-ingestion-warnings, etc. never overflow).
/// * `force_overflow = true` (set upstream by event restrictions) emits the
///   `event_restriction` counter and short-circuits — the limiter is NOT
///   consulted. This runs on both
///   overflowing lanes regardless of limiter presence: the sink honors
///   `force_overflow` on its own (restriction rerouting must survive
///   `OVERFLOW_ENABLED=false`). The counter can over-report in one corner —
///   a forced AI event on Import is counted here but ignored by the sink;
///   only the sink knows the deployment's lanes.
/// * If the lane's limiter is `None`, the event passes through untouched.
/// * If the lane's limiter is `Some`, `is_limited(event.key())` is consulted
///   and the resulting [`OverflowReason`] is stamped, plus the matching
///   counter. `ForceLimited` additionally sets `skip_person_processing =
///   true` for pipeline-level readers of the flag; the sink derives the
///   `force_disable_person_processing` header from the reason itself, so
///   the header does not depend on this stamp.
///
/// Dashboards filter on the `reason` label of `capture_events_rerouted_overflow`,
/// so keep its values stable.
pub fn stamp_overflow_reason(
    events: &mut [ProcessedEvent],
    analytics_limiter: Option<&Arc<OverflowLimiter>>,
    ai_limiter: Option<&Arc<OverflowLimiter>>,
) {
    for event in events.iter_mut() {
        let lane_limiter = match event.metadata.data_type {
            DataType::AnalyticsMain => analytics_limiter,
            DataType::AiEvents => ai_limiter,
            _ => continue,
        };

        if event.metadata.force_overflow {
            counter!(
                "capture_events_rerouted_overflow",
                "reason" => "event_restriction",
            )
            .increment(1);
            continue;
        }

        let Some(limiter) = lane_limiter else {
            continue;
        };

        let event_key = event.event.key();
        match limiter.is_limited(&event_key) {
            OverflowLimiterResult::ForceLimited => {
                counter!(
                    "capture_events_rerouted_overflow",
                    "reason" => "force_limited",
                )
                .increment(1);
                event.metadata.overflow_reason = Some(OverflowReason::ForceLimited);
                // ForceLimited implies person processing is skipped; the sink
                // derives that from the reason itself
                // (`person_processing_disabled`). The flag is stamped
                // alongside so pipeline-level readers of the flag see the
                // same truth.
                event.metadata.skip_person_processing = true;
            }
            OverflowLimiterResult::Limited => {
                counter!(
                    "capture_events_rerouted_overflow",
                    "reason" => "rate_limited",
                )
                .increment(1);
                event.metadata.overflow_reason = Some(OverflowReason::RateLimited {
                    preserve_locality: limiter.should_preserve_locality(),
                });
            }
            OverflowLimiterResult::NotLimited => {}
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::utils::uuid_v7_from_datetime;
    use crate::v0_request::{ProcessedEvent, ProcessedEventMetadata};
    use common_types::CapturedEvent;
    use std::num::NonZeroU32;

    fn build_event(
        data_type: DataType,
        token: &str,
        distinct_id: &str,
        force_overflow: bool,
    ) -> ProcessedEvent {
        let timestamp = chrono::Utc::now();
        let event = CapturedEvent {
            uuid: uuid_v7_from_datetime(timestamp),
            distinct_id: distinct_id.to_string(),
            session_id: None,
            ip: "127.0.0.1".to_string(),
            data: "{}".to_string(),
            now: "2026-04-20T00:00:00Z".to_string(),
            sent_at: None,
            token: token.to_string(),
            event: "test".to_string(),
            timestamp,
            is_cookieless_mode: false,
            historical_migration: false,
        };

        let metadata = ProcessedEventMetadata {
            data_type,
            session_id: None,
            computed_timestamp: None,
            event_name: "test".to_string(),
            force_overflow,
            skip_person_processing: false,
            redirect_to_dlq: false,
            redirect_to_topic: None,
            skip_heatmap_processing: false,
            overflow_reason: None,
            distinct_id_truncated_from: None,
        };

        ProcessedEvent { event, metadata }
    }

    fn build_limiter(
        per_second: u32,
        burst: u32,
        keys_to_reroute: Option<String>,
        preserve_locality: bool,
    ) -> Arc<OverflowLimiter> {
        Arc::new(OverflowLimiter::new(
            NonZeroU32::new(per_second).unwrap(),
            NonZeroU32::new(burst).unwrap(),
            keys_to_reroute,
            preserve_locality,
        ))
    }

    #[test]
    fn force_overflow_short_circuits_without_consulting_limiter() {
        // Use a limiter that would otherwise force-route this event; the
        // short-circuit must skip the limiter call so the stamped reason
        // stays None (force_overflow drives the sink directly without a
        // RateLimited/ForceLimited stamp).
        let limiter = build_limiter(10, 10, Some("phc_t:user".to_string()), false);
        let mut events = vec![build_event(
            DataType::AnalyticsMain,
            "phc_t",
            "user",
            true, // force_overflow
        )];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(
            events[0].metadata.overflow_reason, None,
            "force_overflow short-circuits before the limiter; reason stays None"
        );
        assert!(
            !events[0].metadata.skip_person_processing,
            "force_overflow alone must not flip skip_person_processing"
        );
    }

    #[test]
    fn force_limited_stamps_reason_and_skip_person_processing() {
        let limiter = build_limiter(10, 10, Some("phc_t:user".to_string()), false);
        let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "user", false)];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(
            events[0].metadata.overflow_reason,
            Some(OverflowReason::ForceLimited)
        );
        assert!(
            events[0].metadata.skip_person_processing,
            "ForceLimited must set skip_person_processing alongside the reason"
        );
    }

    #[test]
    fn rate_limited_stamps_preserve_locality_true() {
        // burst=1 means the second event exceeds the budget; preserve_locality
        // is mirrored from the limiter config onto the stamped reason.
        let limiter = build_limiter(1, 1, None, true);
        let mut events = vec![
            build_event(DataType::AnalyticsMain, "phc_t", "u", false),
            build_event(DataType::AnalyticsMain, "phc_t", "u", false),
        ];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(
            events[0].metadata.overflow_reason, None,
            "first event within burst must not be stamped"
        );
        assert_eq!(
            events[1].metadata.overflow_reason,
            Some(OverflowReason::RateLimited {
                preserve_locality: true
            })
        );
        assert!(
            !events[1].metadata.skip_person_processing,
            "RateLimited (non-Force) must not flip skip_person_processing"
        );
    }

    #[test]
    fn rate_limited_stamps_preserve_locality_false() {
        let limiter = build_limiter(1, 1, None, false);
        let mut events = vec![
            build_event(DataType::AnalyticsMain, "phc_t", "u", false),
            build_event(DataType::AnalyticsMain, "phc_t", "u", false),
        ];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(
            events[1].metadata.overflow_reason,
            Some(OverflowReason::RateLimited {
                preserve_locality: false
            })
        );
    }

    #[test]
    fn not_limited_leaves_reason_none() {
        // burst=10, single event; well under the budget.
        let limiter = build_limiter(10, 10, None, false);
        let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", false)];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(events[0].metadata.overflow_reason, None);
    }

    #[test]
    fn non_analytics_main_events_are_skipped() {
        let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
        let mut events = vec![
            build_event(DataType::SnapshotMain, "phc_t", "u", false),
            build_event(DataType::HeatmapMain, "phc_t", "u", false),
            build_event(DataType::ExceptionErrorTracking, "phc_t", "u", false),
            build_event(DataType::ClientIngestionWarning, "phc_t", "u", false),
            build_event(DataType::AnalyticsHistorical, "phc_t", "u", false),
            build_event(DataType::AiEvents, "phc_t", "u", false),
        ];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        for (i, ev) in events.iter().enumerate() {
            assert_eq!(
                ev.metadata.overflow_reason, None,
                "event[{i}] data_type {:?} must be skipped",
                ev.metadata.data_type
            );
            assert!(!ev.metadata.skip_person_processing, "event[{i}]: untouched");
        }
    }

    #[test]
    fn none_limiter_is_a_no_op_for_non_force_overflow_events() {
        let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", false)];

        stamp_overflow_reason(&mut events, None, None);

        assert_eq!(events[0].metadata.overflow_reason, None);
        assert!(!events[0].metadata.skip_person_processing);
    }

    #[test]
    fn none_limiter_still_emits_event_restriction_counter() {
        // force_overflow is independent of the limiter, so the
        // event_restriction short-circuit must still fire (and emit its
        // counter) even when the limiter is absent.
        let mut events = vec![build_event(DataType::AnalyticsMain, "phc_t", "u", true)];

        stamp_overflow_reason(&mut events, None, None);

        assert_eq!(
            events[0].metadata.overflow_reason, None,
            "force_overflow leaves overflow_reason as None"
        );
    }

    #[test]
    fn empty_batch_is_a_no_op() {
        let limiter = build_limiter(10, 10, None, false);
        let mut events: Vec<ProcessedEvent> = Vec::new();
        stamp_overflow_reason(&mut events, Some(&limiter), None);
    }

    #[test]
    fn mixed_batch_stamps_only_analytics_main_entries() {
        // Realistic OTEL-shaped batch: a few AnalyticsMain spans plus one
        // HeatmapMain that snuck in. Only the AnalyticsMain over-budget
        // entries should be stamped.
        let limiter = build_limiter(1, 1, None, true);
        let mut events = vec![
            build_event(DataType::AnalyticsMain, "phc_t", "user_a", false),
            build_event(DataType::HeatmapMain, "phc_t", "user_a", false),
            build_event(DataType::AnalyticsMain, "phc_t", "user_a", false),
        ];

        stamp_overflow_reason(&mut events, Some(&limiter), None);

        assert_eq!(
            events[0].metadata.overflow_reason, None,
            "first AnalyticsMain within burst"
        );
        assert_eq!(
            events[1].metadata.overflow_reason, None,
            "HeatmapMain must be skipped regardless of limiter state"
        );
        assert_eq!(
            events[2].metadata.overflow_reason,
            Some(OverflowReason::RateLimited {
                preserve_locality: true
            }),
            "second AnalyticsMain over budget"
        );
    }

    #[rstest::rstest]
    #[case::limiter_present(true, Some(OverflowReason::ForceLimited), true)]
    #[case::limiter_absent(false, None, false)]
    fn ai_events_stamping_gated_on_limiter_presence(
        #[case] ai_limiter_present: bool,
        #[case] expected_reason: Option<OverflowReason>,
        #[case] expected_skip_person: bool,
    ) {
        let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
        let ai_limiter = ai_limiter_present.then_some(&limiter);
        let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", false)];

        stamp_overflow_reason(&mut events, None, ai_limiter);

        assert_eq!(events[0].metadata.overflow_reason, expected_reason);
        assert_eq!(
            events[0].metadata.skip_person_processing,
            expected_skip_person
        );
    }

    #[test]
    fn ai_events_rate_limited_mirrors_preserve_locality() {
        // The AI limiter's preserve-partition-locality flag is mirrored onto
        // the stamped reason, exactly like the analytics lane.
        let limiter = build_limiter(1, 1, None, true);
        let mut events = vec![
            build_event(DataType::AiEvents, "phc_t", "u", false),
            build_event(DataType::AiEvents, "phc_t", "u", false),
        ];

        stamp_overflow_reason(&mut events, None, Some(&limiter));

        assert_eq!(events[0].metadata.overflow_reason, None);
        assert_eq!(
            events[1].metadata.overflow_reason,
            Some(OverflowReason::RateLimited {
                preserve_locality: true
            })
        );
    }

    /// The two lanes consult separate limiter instances, so one lane
    /// exhausting a key's budget must not stamp the other lane's events for
    /// the same `token:distinct_id` key.
    #[rstest::rstest]
    #[case::analytics_burst_leaves_ai_untouched(DataType::AnalyticsMain, DataType::AiEvents)]
    #[case::ai_burst_leaves_analytics_untouched(DataType::AiEvents, DataType::AnalyticsMain)]
    fn lane_budgets_are_isolated_per_key(
        #[case] bursting_lane: DataType,
        #[case] quiet_lane: DataType,
    ) {
        // burst=1 for both limiters: the bursting lane's second event goes
        // over budget, while the quiet lane's single event (same key) must
        // stay within its own untouched budget.
        let analytics_limiter = build_limiter(1, 1, None, false);
        let ai_limiter = build_limiter(1, 1, None, false);
        let mut events = vec![
            build_event(bursting_lane, "phc_t", "u", false),
            build_event(bursting_lane, "phc_t", "u", false),
            build_event(quiet_lane, "phc_t", "u", false),
        ];

        stamp_overflow_reason(&mut events, Some(&analytics_limiter), Some(&ai_limiter));

        assert_eq!(events[0].metadata.overflow_reason, None);
        assert_eq!(
            events[1].metadata.overflow_reason,
            Some(OverflowReason::RateLimited {
                preserve_locality: false
            }),
            "second {bursting_lane:?} event over budget"
        );
        assert_eq!(
            events[2].metadata.overflow_reason, None,
            "{quiet_lane:?} must not inherit the {bursting_lane:?} lane's exhausted budget"
        );
    }

    #[test]
    fn ai_lane_ignores_analytics_limiter_when_own_limiter_absent() {
        // A missing AI limiter means no rate-limit overflow for the AI lane,
        // even when the analytics limiter would force-route the same key.
        let analytics_limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
        let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", false)];

        stamp_overflow_reason(&mut events, Some(&analytics_limiter), None);

        assert_eq!(events[0].metadata.overflow_reason, None);
    }

    #[test]
    fn ai_events_force_overflow_short_circuits() {
        // Identical to the analytics lane: a restriction-driven
        // force_overflow bypasses the limiter and leaves the reason unset;
        // the sink routes on the flag alone.
        let limiter = build_limiter(10, 10, Some("phc_t:u".to_string()), false);
        let mut events = vec![build_event(DataType::AiEvents, "phc_t", "u", true)];

        stamp_overflow_reason(&mut events, None, Some(&limiter));

        assert_eq!(events[0].metadata.overflow_reason, None);
        assert!(!events[0].metadata.skip_person_processing);
    }
}