:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Pipeline and lane — an event's address, and the one place it is decided.
//!
//! [`resolve`] folds the intent stamped on [`ProcessedEventMetadata`] during
//! processing (restrictions, overflow reasons, the historical flag) through
//! one precedence chain — dlq > custom > historical > overflow > main — and
//! returns an [`AddressDecision`]: the [`Address`] the event publishes to and
//! the customer-facing [`OrderingGuarantee`] the sink realizes as a partition
//! key. The decision is pure — no counters, no headers, no I/O; side effects
//! are implied by the address and applied by the layer that realizes it.
use crate::api::CaptureError;
use crate::ordering::{person_ordering, OrderingGuarantee};
use crate::v0_request::{DataType, OverflowReason, ProcessedEventMetadata};
/// Pipeline from [`DataType`], lane from [`resolve`]. Each pipeline names only
/// the lanes it has, so an address no output backs cannot be built.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum PipelineLane {
Analytics(AnalyticsLane),
Ai(AiLane),
Heatmaps(BasicLane),
Warnings(BasicLane),
ErrorTracking(BasicLane),
Replay(SessionReplayLane),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum AnalyticsLane {
Main,
Overflow,
Historical,
}
/// No historical lane: the AI divert wins over historical rerouting.
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum AiLane {
Main,
Overflow,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum SessionReplayLane {
Main,
Overflow,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum BasicLane {
Main,
}
/// Where an event publishes: a lane of its pipeline, or an admin redirect
/// outside the pipeline-lane model. A redirect carries no pipeline: every
/// pipeline shares one dlq output, and a custom redirect names its own topic.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Address {
Lane(PipelineLane),
Dlq,
Custom(String),
}
/// The pure routing decision for a single event: which address, and which
/// ordering guarantee. Depends only on [`ProcessedEventMetadata`] (stamped
/// upstream by the pipeline) and whether the deployment has an AI overflow
/// lane — the one piece of deployment config that changes a routing decision
/// rather than a topic name. Side effects are not part of the decision: the
/// dlq header set and
/// the reroute counters follow from the address, and the person-processing
/// header follows from
/// [`ProcessedEventMetadata::person_processing_disabled`] — the stamped flag,
/// or a `ForceLimited` reason, which implies the skip on its own.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AddressDecision {
pub address: Address,
pub ordering: OrderingGuarantee,
}
impl AddressDecision {
fn lane(lane: PipelineLane, ordering: OrderingGuarantee) -> Self {
Self {
address: Address::Lane(lane),
ordering,
}
}
/// Both admin redirects key on the event key.
fn redirect(address: Address) -> Self {
Self {
address,
ordering: OrderingGuarantee::PerDistinctId,
}
}
}
/// Decide an event's address from its metadata. DLQ and custom-topic
/// redirects take priority over per-datatype and overflow routing, and a
/// restriction's `force_overflow` takes priority over a stamped
/// `overflow_reason`. Consulted
/// by the sink, which resolves the address to a topic string and the ordering
/// guarantee to a partition key, and applies the address-implied side
/// effects. A replay event with no session id is rejected here, so every
/// returned decision is realizable by the sink.
pub fn resolve(
metadata: &ProcessedEventMetadata,
has_ai_overflow_lane: bool,
) -> Result<AddressDecision, CaptureError> {
if metadata.redirect_to_dlq {
return Ok(AddressDecision::redirect(Address::Dlq));
}
if let Some(ref topic) = metadata.redirect_to_topic {
return Ok(AddressDecision::redirect(Address::Custom(topic.clone())));
}
Ok(match metadata.data_type {
DataType::AnalyticsHistorical => AddressDecision::lane(
// Historical events never overflow — force_overflow and
// overflow_reason are deliberately ignored here.
PipelineLane::Analytics(AnalyticsLane::Historical),
OrderingGuarantee::PerDistinctId,
),
DataType::AnalyticsMain => {
if metadata.force_overflow {
AddressDecision::lane(
PipelineLane::Analytics(AnalyticsLane::Overflow),
person_ordering(metadata.person_processing_disabled()),
)
} else {
match &metadata.overflow_reason {
Some(OverflowReason::ForceLimited) => AddressDecision::lane(
PipelineLane::Analytics(AnalyticsLane::Overflow),
OrderingGuarantee::None,
),
// The person flag alone decides the key here, in both
// directions. A burst keeps its key while person processing
// is on — the overflow consumer updates persons keyed on
// distinct id, so spreading one distinct id across
// partitions turns a hot key into contended person-row
// updates — which makes the locality preference irrelevant
// on this lane. And a key whose person processing is
// already off (the global rate limiter stamps its verdict
// before the overflow limiter overwrites the reason) must
// not get its partition back.
Some(OverflowReason::RateLimited { .. }) => AddressDecision::lane(
PipelineLane::Analytics(AnalyticsLane::Overflow),
person_ordering(metadata.person_processing_disabled()),
),
// ReplayLimited is stamped only by the recordings pipeline,
// so an analytics event cannot carry it — the shared
// OverflowReason enum forces the arm, which treats the
// impossible stamp as unstamped.
Some(OverflowReason::ReplayLimited) | None => AddressDecision::lane(
PipelineLane::Analytics(AnalyticsLane::Main),
person_ordering(metadata.person_processing_disabled()),
),
}
}
}
DataType::AiEvents => {
// The AI lanes route overflow like analytics, except that a burst
// may spread while person processing is on — the AI consumer reads
// persons without writing them, so keyless person-on records cause
// no person-update contention there.
if has_ai_overflow_lane && metadata.force_overflow {
AddressDecision::lane(
PipelineLane::Ai(AiLane::Overflow),
person_ordering(metadata.person_processing_disabled()),
)
} else if has_ai_overflow_lane {
match &metadata.overflow_reason {
Some(OverflowReason::ForceLimited) => AddressDecision::lane(
PipelineLane::Ai(AiLane::Overflow),
OrderingGuarantee::None,
),
Some(OverflowReason::RateLimited {
preserve_locality: true,
}) => AddressDecision::lane(
PipelineLane::Ai(AiLane::Overflow),
// Same precedence as the analytics overflow lane above.
person_ordering(metadata.person_processing_disabled()),
),
Some(OverflowReason::RateLimited {
preserve_locality: false,
}) => AddressDecision::lane(
PipelineLane::Ai(AiLane::Overflow),
OrderingGuarantee::None,
),
// ReplayLimited cannot be stamped on the AI lane either;
// treated as unstamped, as above.
Some(OverflowReason::ReplayLimited) | None => AddressDecision::lane(
PipelineLane::Ai(AiLane::Main),
OrderingGuarantee::PerDistinctId,
),
}
} else {
AddressDecision::lane(
PipelineLane::Ai(AiLane::Main),
OrderingGuarantee::PerDistinctId,
)
}
}
// A ForceOverflow restriction can stamp `force_overflow` on these
// events. Their lane type has no overflow, so the stamp is ignored and
// the event publishes to main.
DataType::ClientIngestionWarning => AddressDecision::lane(
PipelineLane::Warnings(BasicLane::Main),
OrderingGuarantee::PerDistinctId,
),
DataType::HeatmapMain => AddressDecision::lane(
PipelineLane::Heatmaps(BasicLane::Main),
OrderingGuarantee::PerDistinctId,
),
DataType::ExceptionErrorTracking => AddressDecision::lane(
PipelineLane::ErrorTracking(BasicLane::Main),
OrderingGuarantee::PerDistinctId,
),
DataType::SnapshotMain => {
// Partition key is always session_id for replay to keep per-session
// ordering on the overflow lane; a missing id makes the decision
// unrealizable, so it is rejected as part of the decision.
if metadata.session_id.is_none() {
return Err(CaptureError::MissingSessionId);
}
let lane = if metadata.force_overflow
|| matches!(
metadata.overflow_reason,
Some(OverflowReason::ReplayLimited)
) {
SessionReplayLane::Overflow
} else {
SessionReplayLane::Main
};
AddressDecision::lane(PipelineLane::Replay(lane), OrderingGuarantee::PerSession)
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
fn meta(data_type: DataType) -> ProcessedEventMetadata {
ProcessedEventMetadata {
data_type,
session_id: Some("session123".to_string()),
computed_timestamp: None,
event_name: "test_event".to_string(),
force_overflow: false,
skip_person_processing: false,
redirect_to_dlq: false,
redirect_to_topic: None,
skip_heatmap_processing: false,
overflow_reason: None,
distinct_id_truncated_from: None,
}
}
fn lane(lane: PipelineLane) -> Address {
Address::Lane(lane)
}
#[test]
fn dlq_wins_over_custom_topic_and_datatype() {
// redirect_to_dlq set alongside redirect_to_topic and an overflow
// reason: DLQ still wins, keyed on the event key, with the DLQ effect.
let mut m = meta(DataType::AnalyticsMain);
m.redirect_to_dlq = true;
m.redirect_to_topic = Some("custom".to_string());
m.force_overflow = true;
assert_eq!(
resolve(&m, false).unwrap(),
AddressDecision {
address: Address::Dlq,
ordering: OrderingGuarantee::PerDistinctId,
}
);
}
#[test]
fn custom_topic_wins_over_datatype() {
// Custom-topic redirect beats per-datatype/overflow routing (but not DLQ).
let mut m = meta(DataType::AnalyticsMain);
m.redirect_to_topic = Some("my_topic".to_string());
m.force_overflow = true;
assert_eq!(
resolve(&m, false).unwrap(),
AddressDecision {
address: Address::Custom("my_topic".to_string()),
ordering: OrderingGuarantee::PerDistinctId,
}
);
}
#[rstest]
#[case(DataType::AnalyticsMain, PipelineLane::Analytics(AnalyticsLane::Main))]
#[case(
DataType::AnalyticsHistorical,
PipelineLane::Analytics(AnalyticsLane::Historical)
)]
#[case(
DataType::ClientIngestionWarning,
PipelineLane::Warnings(BasicLane::Main)
)]
#[case(DataType::HeatmapMain, PipelineLane::Heatmaps(BasicLane::Main))]
#[case(
DataType::ExceptionErrorTracking,
PipelineLane::ErrorTracking(BasicLane::Main)
)]
#[case(DataType::AiEvents, PipelineLane::Ai(AiLane::Main))]
#[case(DataType::SnapshotMain, PipelineLane::Replay(SessionReplayLane::Main))]
fn per_datatype_addresses(#[case] data_type: DataType, #[case] expected_lane: PipelineLane) {
let m = meta(data_type);
let d = resolve(&m, false).unwrap();
assert_eq!(
d.address,
lane(expected_lane),
"wrong address for {data_type:?}"
);
}
#[rstest]
#[case(
DataType::ClientIngestionWarning,
PipelineLane::Warnings(BasicLane::Main)
)]
#[case(DataType::HeatmapMain, PipelineLane::Heatmaps(BasicLane::Main))]
#[case(
DataType::ExceptionErrorTracking,
PipelineLane::ErrorTracking(BasicLane::Main)
)]
fn single_lane_pipelines_ignore_overflow_intent(
#[case] data_type: DataType,
#[case] main: PipelineLane,
) {
let mut m = meta(data_type);
m.force_overflow = true;
m.overflow_reason = Some(OverflowReason::ForceLimited);
assert_eq!(
resolve(&m, true).unwrap().address,
lane(main),
"overflow intent must not move {data_type:?} off its main lane"
);
}
#[test]
fn analytics_main_overflow_ordering() {
// force_overflow -> overflow lane; key policy follows skip_person.
let mut m = meta(DataType::AnalyticsMain);
m.force_overflow = true;
assert_eq!(
resolve(&m, false).unwrap().ordering,
OrderingGuarantee::PerDistinctId
);
m.skip_person_processing = true;
assert_eq!(
resolve(&m, false).unwrap().ordering,
OrderingGuarantee::None
);
assert_eq!(
resolve(&m, false).unwrap().address,
lane(PipelineLane::Analytics(AnalyticsLane::Overflow))
);
}
#[test]
fn analytics_main_overflow_reason_precedence() {
let base = meta(DataType::AnalyticsMain);
let mut force_limited = base.clone();
force_limited.overflow_reason = Some(OverflowReason::ForceLimited);
assert_eq!(
resolve(&force_limited, false).unwrap(),
AddressDecision {
address: lane(PipelineLane::Analytics(AnalyticsLane::Overflow)),
ordering: OrderingGuarantee::None,
}
);
let mut preserve = base.clone();
preserve.overflow_reason = Some(OverflowReason::RateLimited {
preserve_locality: true,
});
assert_eq!(
resolve(&preserve, false).unwrap().ordering,
OrderingGuarantee::PerDistinctId
);
assert_eq!(
resolve(&preserve, false).unwrap().address,
lane(PipelineLane::Analytics(AnalyticsLane::Overflow))
);
// The locality preference is irrelevant on the analytics lane: a
// person-on burst keeps its key either way, because the overflow
// consumer writes persons keyed on distinct id.
let mut no_preserve = base.clone();
no_preserve.overflow_reason = Some(OverflowReason::RateLimited {
preserve_locality: false,
});
assert_eq!(
resolve(&no_preserve, false).unwrap().ordering,
OrderingGuarantee::PerDistinctId
);
assert_eq!(
resolve(&no_preserve, false).unwrap().address,
lane(PipelineLane::Analytics(AnalyticsLane::Overflow))
);
no_preserve.skip_person_processing = true;
assert_eq!(
resolve(&no_preserve, false).unwrap().ordering,
OrderingGuarantee::None
);
// ReplayLimited cannot be stamped on analytics events (only the
// recordings pipeline produces it); the impossible combination is
// treated as unstamped.
let mut replay = base;
replay.overflow_reason = Some(OverflowReason::ReplayLimited);
assert_eq!(
resolve(&replay, false).unwrap().address,
lane(PipelineLane::Analytics(AnalyticsLane::Main))
);
}
/// The global rate limiter stamps `skip_person_processing` before the
/// overflow limiter runs, and the overflow limiter overwrites the reason it
/// stamped. Without this precedence a key the rate limiter declared too hot
/// would go back to hashing onto a single overflow partition whenever the
/// limiter preserves locality.
#[rstest]
#[case::analytics(
DataType::AnalyticsMain,
PipelineLane::Analytics(AnalyticsLane::Overflow)
)]
#[case::ai(DataType::AiEvents, PipelineLane::Ai(AiLane::Overflow))]
fn person_processing_off_outranks_preserve_locality(
#[case] data_type: DataType,
#[case] overflow: PipelineLane,
) {
let has_ai_overflow_lane = data_type == DataType::AiEvents;
let mut m = meta(data_type);
m.overflow_reason = Some(OverflowReason::RateLimited {
preserve_locality: true,
});
assert_eq!(
resolve(&m, has_ai_overflow_lane).unwrap().ordering,
OrderingGuarantee::PerDistinctId,
"locality is preserved while person processing is on"
);
m.skip_person_processing = true;
assert_eq!(
resolve(&m, has_ai_overflow_lane).unwrap(),
AddressDecision {
address: lane(overflow),
ordering: OrderingGuarantee::None,
}
);
}
#[test]
fn ai_events_overflow_needs_an_overflow_lane() {
let mut m = meta(DataType::AiEvents);
m.force_overflow = true;
assert_eq!(
resolve(&m, false).unwrap(),
AddressDecision {
address: lane(PipelineLane::Ai(AiLane::Main)),
ordering: OrderingGuarantee::PerDistinctId,
}
);
assert_eq!(
resolve(&m, true).unwrap().address,
lane(PipelineLane::Ai(AiLane::Overflow))
);
assert_eq!(
resolve(&m, true).unwrap().ordering,
OrderingGuarantee::PerDistinctId
);
m.skip_person_processing = true;
assert_eq!(resolve(&m, true).unwrap().ordering, OrderingGuarantee::None);
let mut force_limited = meta(DataType::AiEvents);
force_limited.overflow_reason = Some(OverflowReason::ForceLimited);
assert_eq!(
resolve(&force_limited, true).unwrap(),
AddressDecision {
address: lane(PipelineLane::Ai(AiLane::Overflow)),
ordering: OrderingGuarantee::None,
}
);
assert_eq!(
resolve(&force_limited, false).unwrap().address,
lane(PipelineLane::Ai(AiLane::Main))
);
}
#[test]
fn ai_events_default_route_keeps_event_key() {
// skip_person_processing must not null the key on the AI default
// route: v1 only nulls keys for Main/Overflow-shaped destinations.
let mut m = meta(DataType::AiEvents);
m.skip_person_processing = true;
for has_ai_overflow_lane in [false, true] {
assert_eq!(
resolve(&m, has_ai_overflow_lane).unwrap(),
AddressDecision {
address: lane(PipelineLane::Ai(AiLane::Main)),
ordering: OrderingGuarantee::PerDistinctId,
},
"has_ai_overflow_lane={has_ai_overflow_lane}"
);
}
}
#[test]
fn snapshot_routing_keeps_per_session_ordering() {
let mut m = meta(DataType::SnapshotMain);
assert_eq!(
resolve(&m, false).unwrap(),
AddressDecision {
address: lane(PipelineLane::Replay(SessionReplayLane::Main)),
ordering: OrderingGuarantee::PerSession,
}
);
m.force_overflow = true;
assert_eq!(
resolve(&m, false).unwrap().address,
lane(PipelineLane::Replay(SessionReplayLane::Overflow))
);
assert_eq!(
resolve(&m, false).unwrap().ordering,
OrderingGuarantee::PerSession
);
m.force_overflow = false;
m.overflow_reason = Some(OverflowReason::ReplayLimited);
assert_eq!(
resolve(&m, false).unwrap().address,
lane(PipelineLane::Replay(SessionReplayLane::Overflow))
);
}
/// A replay event with no session id has no realizable address — the
/// decision itself rejects, rather than handing the sink a `PerSession`
/// guarantee it cannot key.
#[test]
fn snapshot_without_session_id_is_rejected() {
let mut m = meta(DataType::SnapshotMain);
m.session_id = None;
assert!(matches!(
resolve(&m, false),
Err(CaptureError::MissingSessionId)
));
// A dlq redirect keys on the event key, so it stays realizable
// without a session id.
m.redirect_to_dlq = true;
assert_eq!(resolve(&m, false).unwrap().address, Address::Dlq);
}
}