:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Merge-protocol follower consumers: `person_merge_events` (drain trigger, keyed by P_old) and
//! `cohort_merge_state_transfer` (packaged state, keyed by P_new).
//!
//! Both use [`FollowerConsumer`] generic over a [`FollowerRoute`]. Followers never `subscribe()`:
//! each is assigned the partitions this pod owns, like the events consumer.
use std::collections::{HashMap, HashSet};
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use lifecycle::Handle;
use metrics::{counter, histogram};
use rdkafka::consumer::{CommitMode, StreamConsumer};
use rdkafka::message::Message;
use tracing::{debug, info, warn};
use crate::cascade::CascadeMessage;
use crate::consumers::events::{fsync_then_commit, DrainWait, EventDispatcher, RECV_ERROR_BACKOFF};
use crate::merge::transfer::{MergeStateTransfer, PersonMergeEvent};
use crate::observability::metrics::{
COHORT_STREAM_CASCADES_CONSUMED, COHORT_STREAM_CASCADES_CONSUME_BATCH_SIZE,
COHORT_STREAM_CASCADE_DESERIALIZE_ERRORS, COHORT_STREAM_KAFKA_RECV_ERRORS,
COHORT_STREAM_MERGES_CONSUMED, COHORT_STREAM_MERGES_CONSUME_BATCH_SIZE,
COHORT_STREAM_MERGE_DESERIALIZE_ERRORS, COHORT_STREAM_TRANSFERS_CONSUMED,
COHORT_STREAM_TRANSFERS_CONSUME_BATCH_SIZE, COHORT_STREAM_TRANSFER_DESERIALIZE_ERRORS,
};
use crate::partitions::offset_tracker::OffsetTracker;
use crate::partitions::stamp::CommitStamp;
use crate::workers::MergeWorkerDeps;
/// One merge trigger consumed from `person_merge_events`, paired with its commit coordinates.
#[derive(Debug)]
pub struct ConsumedMerge {
pub event: PersonMergeEvent,
pub partition: i32,
pub offset: i64,
}
/// One state transfer consumed from `cohort_merge_state_transfer`, paired with its commit
/// coordinates. Replay idempotence is keyed by the transfer's source merge-message coordinates
/// (`source_partition` + `source_offset`).
#[derive(Debug)]
pub struct ConsumedTransfer {
pub transfer: MergeStateTransfer,
pub partition: i32,
pub offset: i64,
}
/// One cohort flip consumed from `cohort_cascade_events`, paired with its commit coordinates.
/// Replay idempotence is the handler's own bit-flip check (no stored offset field).
#[derive(Debug)]
pub struct ConsumedCascade {
pub message: CascadeMessage,
pub partition: i32,
pub offset: i64,
}
/// How a follower topic's payload decodes and which dispatcher entry point routes the batch.
#[async_trait]
pub trait FollowerRoute: Send + Sync + 'static {
type Consumed: Send + 'static;
const KIND: &'static str;
const CONSUMED_TOTAL: &'static str;
const DESERIALIZE_ERRORS_TOTAL: &'static str;
const CONSUME_BATCH_SIZE: &'static str;
/// The route-specific commit tracker. Each route selects its own so commits never cross topics.
fn tracker(deps: &MergeWorkerDeps) -> &Arc<OffsetTracker>;
/// Decode one payload into its consumed envelope.
fn decode(
payload: &[u8],
partition: i32,
offset: i64,
) -> Result<Self::Consumed, serde_json::Error>;
/// Hand a decoded batch to the dispatcher.
async fn dispatch(dispatcher: &EventDispatcher, batch: Vec<Self::Consumed>);
}
/// P_old-keyed merge triggers into [`EventDispatcher::dispatch_merges`].
pub struct MergeRoute;
#[async_trait]
impl FollowerRoute for MergeRoute {
type Consumed = ConsumedMerge;
const KIND: &'static str = "person_merge_events";
const CONSUMED_TOTAL: &'static str = COHORT_STREAM_MERGES_CONSUMED;
const DESERIALIZE_ERRORS_TOTAL: &'static str = COHORT_STREAM_MERGE_DESERIALIZE_ERRORS;
const CONSUME_BATCH_SIZE: &'static str = COHORT_STREAM_MERGES_CONSUME_BATCH_SIZE;
fn tracker(deps: &MergeWorkerDeps) -> &Arc<OffsetTracker> {
&deps.merge_tracker
}
fn decode(
payload: &[u8],
partition: i32,
offset: i64,
) -> Result<Self::Consumed, serde_json::Error> {
PersonMergeEvent::decode(payload).map(|event| ConsumedMerge {
event,
partition,
offset,
})
}
async fn dispatch(dispatcher: &EventDispatcher, batch: Vec<Self::Consumed>) {
dispatcher.dispatch_merges(batch).await;
}
}
/// P_new-keyed packaged drains into [`EventDispatcher::dispatch_transfers`].
pub struct TransferRoute;
#[async_trait]
impl FollowerRoute for TransferRoute {
type Consumed = ConsumedTransfer;
const KIND: &'static str = "cohort_merge_state_transfer";
const CONSUMED_TOTAL: &'static str = COHORT_STREAM_TRANSFERS_CONSUMED;
const DESERIALIZE_ERRORS_TOTAL: &'static str = COHORT_STREAM_TRANSFER_DESERIALIZE_ERRORS;
const CONSUME_BATCH_SIZE: &'static str = COHORT_STREAM_TRANSFERS_CONSUME_BATCH_SIZE;
fn tracker(deps: &MergeWorkerDeps) -> &Arc<OffsetTracker> {
&deps.transfer_tracker
}
fn decode(
payload: &[u8],
partition: i32,
offset: i64,
) -> Result<Self::Consumed, serde_json::Error> {
MergeStateTransfer::decode(payload).map(|transfer| ConsumedTransfer {
transfer,
partition,
offset,
})
}
async fn dispatch(dispatcher: &EventDispatcher, batch: Vec<Self::Consumed>) {
dispatcher.dispatch_transfers(batch).await;
}
}
/// `(team, person)`-keyed cohort flips into [`EventDispatcher::dispatch_cascade`].
pub struct CascadeRoute;
#[async_trait]
impl FollowerRoute for CascadeRoute {
type Consumed = ConsumedCascade;
const KIND: &'static str = "cohort_cascade_events";
const CONSUMED_TOTAL: &'static str = COHORT_STREAM_CASCADES_CONSUMED;
const DESERIALIZE_ERRORS_TOTAL: &'static str = COHORT_STREAM_CASCADE_DESERIALIZE_ERRORS;
const CONSUME_BATCH_SIZE: &'static str = COHORT_STREAM_CASCADES_CONSUME_BATCH_SIZE;
fn tracker(deps: &MergeWorkerDeps) -> &Arc<OffsetTracker> {
&deps.cascade_tracker
}
fn decode(
payload: &[u8],
partition: i32,
offset: i64,
) -> Result<Self::Consumed, serde_json::Error> {
serde_json::from_slice::<CascadeMessage>(payload).map(|message| ConsumedCascade {
message,
partition,
offset,
})
}
async fn dispatch(dispatcher: &EventDispatcher, batch: Vec<Self::Consumed>) {
dispatcher.dispatch_cascade(batch).await;
}
}
pub struct FollowerConsumer<R: FollowerRoute> {
consumer: Arc<StreamConsumer>,
topic: String,
stamp: CommitStamp,
dispatcher: Arc<EventDispatcher>,
handle: Handle,
recv_batch_size: usize,
recv_batch_timeout: Duration,
offset_commit_interval: Duration,
_route: PhantomData<R>,
}
impl<R: FollowerRoute> FollowerConsumer<R> {
#[allow(clippy::too_many_arguments)]
pub fn new(
consumer: Arc<StreamConsumer>,
topic: String,
stamp: CommitStamp,
dispatcher: Arc<EventDispatcher>,
handle: Handle,
recv_batch_size: usize,
recv_batch_timeout: Duration,
offset_commit_interval: Duration,
) -> Self {
Self {
consumer,
topic,
stamp,
dispatcher,
handle,
recv_batch_size,
recv_batch_timeout,
offset_commit_interval,
_route: PhantomData,
}
}
fn tracker(&self) -> &Arc<OffsetTracker> {
R::tracker(self.dispatcher.merge_deps())
}
pub async fn process(self) {
let _guard = self.handle.process_scope();
info!(topic = %self.topic, "follower consume loop starting");
let mut commit_deadline = tokio::time::Instant::now() + self.offset_commit_interval;
let owned_at_shutdown = loop {
tokio::select! {
biased;
_ = self.handle.shutdown_recv() => {
info!(topic = %self.topic, "shutdown signal received, stopping follower consume loop");
break self.dispatcher.owned_set();
}
outcome = self.consume_batch() => {
self.handle_outcome(outcome).await;
let now = tokio::time::Instant::now();
if now >= commit_deadline {
fsync_then_commit(
self.dispatcher.handle(),
&self.consumer,
&self.stamp,
self.owned_committable_offsets(),
&self.topic,
CommitMode::Async,
)
.await;
commit_deadline = now + self.offset_commit_interval;
}
}
}
};
let offsets = committable_after_drain(
self.tracker(),
&self.dispatcher,
owned_at_shutdown,
&self.topic,
)
.await;
fsync_then_commit(
self.dispatcher.handle(),
&self.consumer,
&self.stamp,
offsets,
&self.topic,
CommitMode::Sync,
)
.await;
info!(topic = %self.topic, "follower consume loop stopped");
}
async fn handle_outcome(&self, outcome: FollowerOutcome<R::Consumed>) {
histogram!(R::CONSUME_BATCH_SIZE).record(outcome.messages.len() as f64);
if !outcome.messages.is_empty() {
counter!(R::CONSUMED_TOTAL).increment(outcome.messages.len() as u64);
}
if outcome.deserialize_errors > 0 {
counter!(R::DESERIALIZE_ERRORS_TOTAL).increment(outcome.deserialize_errors);
}
// Follower keeps the blocking dispatch on purpose: merges/transfers/cascades are low-volume, so
// a full channel here can await a drain without risking the heartbeat — unlike the events
// consumer's non-blocking, partition-pausing path (`EventDispatcher::dispatch_events_nonblocking`).
// Port that path here if any of these topics ever approaches events-topic volume.
R::dispatch(&self.dispatcher, outcome.messages).await;
if outcome.transport_error {
tokio::time::sleep(RECV_ERROR_BACKOFF).await;
} else {
self.handle.report_healthy();
}
}
async fn consume_batch(&self) -> FollowerOutcome<R::Consumed> {
let mut outcome = FollowerOutcome {
messages: Vec::with_capacity(self.recv_batch_size),
deserialize_errors: 0,
transport_error: false,
};
tokio::select! {
_ = tokio::time::sleep(self.recv_batch_timeout) => {}
_ = async {
while outcome.messages.len() < self.recv_batch_size {
match self.consumer.recv().await {
Ok(message) => {
let partition = message.partition();
let offset = message.offset();
let Some(payload) = message.payload() else {
outcome.deserialize_errors += 1;
debug!(
topic = %self.topic, partition, offset,
"skipping follower message with empty payload",
);
continue;
};
match R::decode(payload, partition, offset) {
Ok(consumed) => outcome.messages.push(consumed),
Err(err) => {
outcome.deserialize_errors += 1;
debug!(
topic = %self.topic, partition, offset, error = %err,
"skipping undeserializable follower message",
);
}
}
}
Err(err) => {
outcome.transport_error = true;
counter!(COHORT_STREAM_KAFKA_RECV_ERRORS).increment(1);
warn!(topic = %self.topic, error = %err, "kafka recv error while consuming follower topic");
break;
}
}
}
} => {}
}
outcome
}
fn owned_committable_offsets(&self) -> HashMap<i32, i64> {
owned_committable_offsets(self.tracker(), &self.dispatcher)
}
}
struct FollowerOutcome<T> {
messages: Vec<T>,
deserialize_errors: u64,
transport_error: bool,
}
/// Bounds a follower's final wait for the events consumer's worker drain. It exceeds the
/// consumer's 30 s shutdown window and stays inside each follower's 45 s window (both in
/// `main.rs`), so a follower outlives the drain and still has time to commit.
pub(crate) const FOLLOWER_DRAIN_WAIT: Duration = Duration::from_secs(35);
/// A tracker's committable offsets restricted to the partitions this pod currently owns.
pub(crate) fn owned_committable_offsets(
tracker: &OffsetTracker,
dispatcher: &EventDispatcher,
) -> HashMap<i32, i64> {
restrict_to_owned(tracker.committable_offsets(), &dispatcher.owned_set())
}
/// A follower's final commit: what the workers finished, including during the shutdown drain, for
/// the partitions this pod owned when that drain ended. A drain that outlasts
/// [`FOLLOWER_DRAIN_WAIT`] falls back to the ownership the follower saw when shutdown began.
pub(crate) async fn committable_after_drain(
tracker: &OffsetTracker,
dispatcher: &EventDispatcher,
owned_at_shutdown: HashSet<i32>,
topic: &str,
) -> HashMap<i32, i64> {
match dispatcher.wait_for_worker_drain(FOLLOWER_DRAIN_WAIT).await {
DrainWait::Drained(drained) => {
restrict_to_owned(tracker.committable_offsets(), drained.partitions())
}
DrainWait::TimedOut => {
warn!(
topic,
"worker drain outlasted the follower's wait; committing against the ownership seen at shutdown",
);
restrict_to_owned(tracker.committable_offsets(), &owned_at_shutdown)
}
}
}
fn restrict_to_owned(offsets: HashMap<i32, i64>, owned: &HashSet<i32>) -> HashMap<i32, i64> {
offsets
.into_iter()
.filter(|(partition, _)| owned.contains(partition))
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use uuid::Uuid;
use crate::merge::transfer::{TransferLeaf, MERGE_EVENT_SCHEMA_VERSION};
use crate::stage1::key::LeafStateKey;
use crate::stage1::state::{AppliedOffsets, Stage1State, StatefulRecord};
fn merge_event() -> PersonMergeEvent {
PersonMergeEvent {
team_id: 42,
old_person_uuid: Uuid::from_u128(0xAAAA),
new_person_uuid: Uuid::from_u128(0xBBBB),
merged_at_ms: 1_716_800_000_000,
schema_version: MERGE_EVENT_SCHEMA_VERSION,
}
}
fn transfer() -> MergeStateTransfer {
MergeStateTransfer {
team_id: 42,
old_person_uuid: Uuid::from_u128(0xAAAA),
new_person_uuid: Uuid::from_u128(0xBBBB),
merged_at_ms: 1_716_800_000_000,
source_partition: 17,
source_offset: 12_345,
leaves: vec![TransferLeaf::new(
LeafStateKey([0xAB; 16]),
StatefulRecord::new(
Stage1State::BehavioralSingle {
has_match: true,
last_event_at_ms: 1_700_000_000_000,
earliest_eviction_at_ms: i64::MAX,
},
AppliedOffsets::default(),
),
)],
membership_registers: vec![],
forward_hops: 0,
person_dedup: None,
}
}
#[test]
fn merge_route_decodes_an_envelope_with_its_commit_coords() {
let event = merge_event();
let consumed = MergeRoute::decode(&event.encode(), 17, 12_345).unwrap();
assert_eq!(consumed.event, event);
assert_eq!(consumed.partition, 17);
assert_eq!(consumed.offset, 12_345);
}
#[test]
fn transfer_route_decodes_an_envelope_with_its_commit_coords() {
let wire = transfer();
let consumed = TransferRoute::decode(&wire.encode(), 9, 77).unwrap();
assert_eq!(consumed.transfer, wire);
assert_eq!(consumed.partition, 9);
assert_eq!(consumed.offset, 77);
}
fn cascade_message() -> CascadeMessage {
crate::cascade::first_cascade(
crate::producer::CohortMembershipChange {
team_id: 42,
cohort_id: 91204,
person_id: Uuid::from_u128(0xCAFE).to_string(),
last_updated: "2026-05-26 12:34:56.789000".to_string(),
status: crate::producer::MembershipStatus::Entered,
origin: None,
run_id: None,
},
123,
)
}
#[test]
fn cascade_route_decodes_an_envelope_with_its_commit_coords() {
let wire = cascade_message();
let consumed = CascadeRoute::decode(&serde_json::to_vec(&wire).unwrap(), 5, 88).unwrap();
assert_eq!(consumed.message, wire);
assert_eq!(consumed.partition, 5);
assert_eq!(consumed.offset, 88);
}
#[test]
fn garbage_payloads_decode_to_err_for_every_route() {
assert!(MergeRoute::decode(b"not json", 0, 0).is_err());
assert!(TransferRoute::decode(&[], 0, 0).is_err());
assert!(CascadeRoute::decode(b"not json", 0, 0).is_err());
}
#[test]
fn each_route_selects_its_own_tracker_from_the_merge_deps() {
let deps = MergeWorkerDeps::capture();
assert!(
Arc::ptr_eq(MergeRoute::tracker(&deps), &deps.merge_tracker),
"merge route must commit via the merge tracker"
);
assert!(
Arc::ptr_eq(TransferRoute::tracker(&deps), &deps.transfer_tracker),
"transfer route must commit via the transfer tracker"
);
assert!(
Arc::ptr_eq(CascadeRoute::tracker(&deps), &deps.cascade_tracker),
"cascade route must commit via the cascade tracker"
);
}
#[test]
fn restrict_to_owned_keeps_only_owned_partitions() {
let offsets: HashMap<i32, i64> = [(0, 10), (3, 30), (7, 70)].into_iter().collect();
let restricted = restrict_to_owned(offsets, &HashSet::from([3, 7]));
assert_eq!(restricted.len(), 2);
assert_eq!(restricted.get(&0), None, "revoked partition is dropped");
assert_eq!(restricted.get(&3), Some(&30));
assert_eq!(restricted.get(&7), Some(&70));
}
#[test]
fn restrict_to_owned_with_no_owned_partitions_is_empty() {
let offsets: HashMap<i32, i64> = [(0, 10)].into_iter().collect();
assert!(restrict_to_owned(offsets, &HashSet::new()).is_empty());
}
}