Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

31.4 KB
Raw Blame
//! Pipelined consume → filter → re-key → produce loop for the input topic.
//!
//! Drops events with no `person_id`, skips teams with no realtime cohorts, then re-keys the
//! survivors by `(team_id, person_id)`, gating on the realtime-cohort filter catalog.
//!
//! Consume, produce-ack, and offset-commit are decoupled: the single owner task enqueues
//! survivors without awaiting delivery, resolves acks as they arrive, and periodically commits
//! the [`Ledger`]'s per-partition watermarks through `spawn_blocking`, so WarpStream's
//! ~250–500ms produce acks never throttle consumption.

use std::future::Future;
use std::pin::Pin;
use std::time::{Duration, Instant};

use common_kafka::kafka_consumer::{Offset, RecvErr, SingleTopicConsumer};
use futures::stream::FuturesUnordered;
use futures::StreamExt;
use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
use rdkafka::error::KafkaError;
use serde::{de::IgnoredAny, Deserialize};
use tokio::task::JoinError;
use tracing::{debug, error, info, warn};

use crate::event::{personless_mode, CohortStreamEvent, ForwardFields, SourcePartitionRange};
use crate::filter_team_index::TeamGate;
use crate::ledger::{
    DeliveryOutcome, Ledger, NextOffset, Observation, Resolution, SourceOffset, SourcePartition,
};
use crate::observability::metrics::{
    ACTIVE_TEAMS, COMMITS, COMMIT_ERRORS, EVENTS_ABANDONED, EVENTS_CONSUMED,
    EVENTS_DROPPED_NO_PERSON_ID, EVENTS_FORWARDED, EVENTS_PERSON_PAYLOAD_DROPPED,
    EVENTS_SKIPPED_TEAM_GATE, EVENTS_UNPARSEABLE, FORWARDS_ENQUEUED, FORWARDS_INFLIGHT,
    LEDGER_PARTITIONS, PRODUCE_ACK_SECONDS, PRODUCE_ERRORS, PRODUCE_QUEUE_FULL, UNCOMMITTED_EVENTS,
};
use crate::producer::{CohortStreamProducer, EnqueueError};

/// Bounds the idle select wait so every loop iteration (and thus the liveness heartbeat) happens
/// at least this often on a quiet topic.
const RECV_TIMEOUT: Duration = Duration::from_millis(500);
const QUEUE_FULL_BACKOFF_CAP: Duration = Duration::from_secs(2);
/// Intake pause after a kafka-level recv error, so a persistently erroring consumer doesn't
/// hot-loop.
const RECV_ERROR_BACKOFF: Duration = Duration::from_secs(1);
/// No successful commit for this many intervals while committable work exists ⇒ stop
/// heartbeating so the stall detector restarts the pod.
const COMMIT_STALENESS_FACTOR: u32 = 3;
/// Every step of the shutdown drain shares this one budget, sized inside the consumer's 15s
/// graceful window with slack for process-exit overhead.
const SHUTDOWN_DRAIN_BUDGET: Duration = Duration::from_secs(14);
/// librdkafka-internal bound on the shutdown flush; the shared budget applies on top.
const SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
/// Floor of the shared budget held back for the final Sync commit — the step that turns drained
/// work into committed progress must not be starved by a slow flush or ack drain. The commit may
/// run for longer (up to the full budget) if earlier steps finish early; this only reserves the
/// minimum.
const SHUTDOWN_COMMIT_RESERVE: Duration = Duration::from_secs(4);

/// Only the fields the team gate needs. Every undeclared field (notably the large
/// `properties` / `person_properties` / `groupN_properties` JSON-string blobs) is skipped by
/// `serde_json` without unescaping or allocating, instead of being materialized.
#[derive(Deserialize)]
struct GateFields {
    team_id: i32,
    /// `IgnoredAny` detects presence without allocating the value: absent and `null` deserialize to
    /// `None`, any present non-null value to `Some`.
    #[serde(default)]
    person_id: Option<IgnoredAny>,
}

#[derive(Debug, PartialEq, Eq)]
enum GateOutcome {
    Forward,
    DropNoPersonId,
    SkipTeamGate,
}

/// `person_id` is checked first: a missing routing key is attributed to `DropNoPersonId` even when
/// the team would also be skipped.
fn classify_gate(gate: &GateFields, team_gate: &TeamGate) -> GateOutcome {
    if gate.person_id.is_none() {
        return GateOutcome::DropNoPersonId;
    }
    if !team_gate.passes(gate.team_id) {
        return GateOutcome::SkipTeamGate;
    }
    GateOutcome::Forward
}

/// `Forward` is boxed to keep the enum small: it is the only large variant and is built only for
/// survivors.
#[derive(Debug)]
enum Decoded {
    Forward(Box<ForwardFields>),
    DropNoPersonId,
    SkipTeamGate,
    /// Serde failure at either phase — counted and settled, never an `Err`. Infallibility matters:
    /// `recv_with` auto-stores offsets on decode errors, and a stored poison-pill offset could be
    /// committed past unacked forwards by an auto-commit path. With this variant the decode-error
    /// auto-store is unreachable; only the empty-payload branch still auto-stores, and stored
    /// offsets stay inert because auto-commit is off (see `Config::build_consumer_config`). The
    /// ledger owns every observed offset.
    Unparseable,
}

/// Cheap gate parse for every event; survivors get a second parse, over the same bytes, of only the
/// fields the envelope forwards.
fn decode_gated(payload: &[u8], team_gate: &TeamGate) -> Decoded {
    let Ok(gate) = serde_json::from_slice::<GateFields>(payload) else {
        return Decoded::Unparseable;
    };
    match classify_gate(&gate, team_gate) {
        GateOutcome::DropNoPersonId => Decoded::DropNoPersonId,
        GateOutcome::SkipTeamGate => Decoded::SkipTeamGate,
        GateOutcome::Forward => serde_json::from_slice::<Box<ForwardFields>>(payload)
            .map_or(Decoded::Unparseable, Decoded::Forward),
    }
}

/// Resolved delivery of one forwarded event, carried back to the pipeline task by an
/// [`AckFuture`].
struct DeliveryAck {
    partition: SourcePartition,
    offset: SourceOffset,
    sent_at: Instant,
    result: Result<(), KafkaError>,
}

type AckFuture = Pin<Box<dyn Future<Output = DeliveryAck> + Send>>;
type CommitResult = Result<Result<(), KafkaError>, JoinError>;
type CommitFuture =
    Pin<Box<dyn Future<Output = (Vec<(SourcePartition, NextOffset)>, CommitResult)> + Send>>;

#[derive(Debug, Clone, Copy)]
pub struct ShufflerSettings {
    pub source_partitions: SourcePartitionRange,
    pub max_inflight_forwards: usize,
    pub commit_interval: Duration,
    pub queue_full_backoff: Duration,
}

/// A gate survivor whose enqueue hit `QueueFull`, parked until the producer queue drains. Its
/// offset is not observed by the ledger until the enqueue lands, so a crash while parked simply
/// replays it.
struct PendingForward {
    event: CohortStreamEvent,
    partition: SourcePartition,
    offset: SourceOffset,
}

/// Whether the recv arm may consume. Pausing is a hard ordering gate — a parked survivor must
/// reach the producer before any newer event for the same `(team, person)` — but only intake
/// stops: acks, commits, shutdown, and the liveness heartbeat keep flowing because the loop
/// itself never sleeps.
enum Intake {
    Open,
    /// `QueueFull`: re-enqueue the parked forward at `resume_at`, doubling `backoff` (capped)
    /// on each further failure. Boxed to keep the always-carried `Open` variant word-sized.
    RetryForward {
        forward: Box<PendingForward>,
        resume_at: tokio::time::Instant,
        backoff: Duration,
    },
    /// Transient kafka-level recv error; resume consuming at `resume_at`. Named for its trigger
    /// (the recv error) to distinguish it from [`Intake::RetryForward`], which carries its own
    /// `backoff`.
    RecvBackoff {
        resume_at: tokio::time::Instant,
    },
}

impl Intake {
    fn is_open(&self) -> bool {
        matches!(self, Intake::Open)
    }

    /// Only meaningful when paused; the select arm awaiting this is guarded on `!is_open()`.
    fn resume_at(&self) -> tokio::time::Instant {
        match self {
            Intake::Open => tokio::time::Instant::now(),
            Intake::RetryForward { resume_at, .. } | Intake::RecvBackoff { resume_at } => {
                *resume_at
            }
        }
    }
}

pub struct EventShuffler {
    consumer: SingleTopicConsumer,
    producer: CohortStreamProducer,
    team_gate: TeamGate,
    handle: Handle,
    settings: ShufflerSettings,
}

impl EventShuffler {
    pub fn new(
        consumer: SingleTopicConsumer,
        producer: CohortStreamProducer,
        team_gate: TeamGate,
        handle: Handle,
        settings: ShufflerSettings,
    ) -> Self {
        Self {
            consumer,
            producer,
            team_gate,
            handle,
            settings,
        }
    }

    /// The single pipeline task: owns the consumer, producer, [`Ledger`], [`Intake`] gate,
    /// pending ack futures, and the one-slot commit task. Backpressure is the recv guard — at
    /// the in-flight cap or while intake is paused the recv arm is disabled but acks and commits
    /// keep draining, and every `DeliveryFuture` resolves within `message.timeout.ms` regardless
    /// of broker state, so the loop cannot deadlock and never stops heartbeating on its own.
    pub async fn process(self) {
        let _guard = self.handle.process_scope();
        info!("shuffler pipeline starting");

        let mut ledger = Ledger::default();
        let mut pending_acks: FuturesUnordered<AckFuture> = FuturesUnordered::new();
        let mut pending_commit: FuturesUnordered<CommitFuture> = FuturesUnordered::new();
        let mut intake = Intake::Open;
        let mut commit_in_flight = false;
        let mut commit_tick = tokio::time::interval(self.settings.commit_interval);
        commit_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
        let staleness_threshold = self.settings.commit_interval * COMMIT_STALENESS_FACTOR;
        let mut last_commit_ok = Instant::now();

        loop {
            tokio::select! {
                biased;

                _ = self.handle.shutdown_recv() => {
                    info!("shutdown signal received, stopping intake");
                    break;
                }

                Some(ack) = pending_acks.next() => {
                    record_delivery(&mut ledger, ack);
                }

                Some((offsets, result)) = pending_commit.next() => {
                    commit_in_flight = false;
                    if apply_commit_result(&mut ledger, offsets, result) {
                        last_commit_ok = Instant::now();
                    }
                }

                _ = commit_tick.tick(), if !commit_in_flight => {
                    if let Some(commit) = self.start_commit(&mut ledger) {
                        pending_commit.push(commit);
                        commit_in_flight = true;
                    }
                }

                _ = tokio::time::sleep_until(intake.resume_at()), if !intake.is_open() => {
                    intake = match std::mem::replace(&mut intake, Intake::Open) {
                        Intake::RetryForward { forward, backoff, .. } => {
                            self.try_enqueue(forward, backoff, &mut ledger, &mut pending_acks)
                        }
                        Intake::Open | Intake::RecvBackoff { .. } => Intake::Open,
                    };
                }

                // Fail-closed until the team gate loads once: don't consume — and advance
                // offsets past — events while the gate would forward nothing.
                recv = tokio::time::timeout(
                    RECV_TIMEOUT,
                    self.consumer.recv_with(|payload| Ok(decode_gated(payload, &self.team_gate))),
                ), if intake.is_open()
                    && ledger.in_flight() < self.settings.max_inflight_forwards
                    && self.team_gate.is_loaded() =>
                {
                    match recv {
                        Ok(Ok((decoded, offset))) => {
                            intake = self.intake(&mut ledger, &mut pending_acks, decoded, &offset);
                        }
                        Ok(Err(err)) => match err {
                            // Auto-stored by recv_with, but inert: this loop never commits
                            // stored offsets, only explicit ledger watermarks. Worst case at a
                            // partition tail: commit stays one message short → one-message
                            // replay on restart.
                            RecvErr::Empty => debug!("skipped message with empty payload"),
                            RecvErr::Serde(e) => {
                                warn!(error = %e, "unreachable: infallible decode reported serde error");
                            }
                            RecvErr::Kafka(e) => {
                                warn!(error = %e, "kafka recv error; pausing intake");
                                intake = Intake::RecvBackoff {
                                    resume_at: tokio::time::Instant::now() + RECV_ERROR_BACKOFF,
                                };
                            }
                        },
                        Err(_elapsed) => {} // idle poll; heartbeat below keeps liveness fresh
                    }
                }
            }

            // Heartbeat gated by commit freshness: a wedged committer with committable work must
            // trip the stall detector, but an idle topic (nothing to commit) stays healthy.
            let commit_is_stale = last_commit_ok.elapsed() > staleness_threshold;
            if should_heartbeat(commit_is_stale, ledger.has_committable()) {
                self.handle.report_healthy();
            }
        }

        self.drain_and_commit(ledger, pending_acks, pending_commit, intake)
            .await;
        info!("shuffler pipeline stopped");
    }

    fn intake(
        &self,
        ledger: &mut Ledger,
        pending_acks: &mut FuturesUnordered<AckFuture>,
        decoded: Decoded,
        offset: &Offset,
    ) -> Intake {
        counter!(EVENTS_CONSUMED).increment(1);
        let partition = SourcePartition(offset.partition());
        let source_offset = SourceOffset(offset.get_value());
        // Returning before the ledger observes the message leaves its partition uncommitted, so a
        // fixed build reads the message again.
        let Some(envelope_partition) = self
            .settings
            .source_partitions
            .envelope_partition(partition)
        else {
            self.handle.signal_failure(format!(
                "input partition {} has no source_partition in this topic's range",
                partition.0
            ));
            return Intake::Open;
        };

        let fields = match decoded {
            Decoded::Forward(fields) => fields,
            Decoded::DropNoPersonId => {
                counter!(EVENTS_DROPPED_NO_PERSON_ID).increment(1);
                ledger.observe(partition, source_offset, Observation::Settled);
                return Intake::Open;
            }
            Decoded::SkipTeamGate => {
                counter!(EVENTS_SKIPPED_TEAM_GATE).increment(1);
                ledger.observe(partition, source_offset, Observation::Settled);
                return Intake::Open;
            }
            Decoded::Unparseable => {
                counter!(EVENTS_UNPARSEABLE).increment(1);
                ledger.observe(partition, source_offset, Observation::Settled);
                return Intake::Open;
            }
        };

        if let Some(person_mode) = personless_mode(fields.person_mode) {
            counter!(EVENTS_PERSON_PAYLOAD_DROPPED, "person_mode" => person_mode).increment(1);
        }
        let forward = Box::new(PendingForward {
            event: CohortStreamEvent::from_forward(*fields, envelope_partition, source_offset.0),
            partition,
            offset: source_offset,
        });
        self.try_enqueue(
            forward,
            self.settings.queue_full_backoff,
            ledger,
            pending_acks,
        )
    }

    /// One non-blocking enqueue attempt. `QueueFull` parks the forward for a timer-arm retry
    /// after `backoff`: the queue drains independently via librdkafka's delivery thread, and
    /// `message.timeout.ms` bounds how long it can stay full even with a dead broker.
    fn try_enqueue(
        &self,
        forward: Box<PendingForward>,
        backoff: Duration,
        ledger: &mut Ledger,
        pending_acks: &mut FuturesUnordered<AckFuture>,
    ) -> Intake {
        match self.producer.enqueue(&forward.event) {
            Ok(delivery) => {
                counter!(FORWARDS_ENQUEUED).increment(1);
                let PendingForward {
                    partition, offset, ..
                } = *forward;
                ledger.observe(partition, offset, Observation::InFlight);
                let sent_at = Instant::now();
                pending_acks.push(Box::pin(async move {
                    let result = match delivery.await {
                        Ok(Ok(_)) => Ok(()),
                        Ok(Err((err, _message))) => Err(err),
                        Err(_canceled) => Err(KafkaError::Canceled),
                    };
                    DeliveryAck {
                        partition,
                        offset,
                        sent_at,
                        result,
                    }
                }));
                Intake::Open
            }
            Err(EnqueueError::QueueFull) => {
                counter!(PRODUCE_QUEUE_FULL).increment(1);
                Intake::RetryForward {
                    resume_at: tokio::time::Instant::now() + backoff,
                    backoff: (backoff * 2).min(QUEUE_FULL_BACKOFF_CAP),
                    forward,
                }
            }
            Err(EnqueueError::Fatal(err)) => {
                counter!(PRODUCE_ERRORS).increment(1);
                counter!(EVENTS_ABANDONED).increment(1);
                warn!(error = %err, "fatal enqueue error; abandoning event");
                ledger.observe(forward.partition, forward.offset, Observation::Settled);
                Intake::Open
            }
        }
    }

    /// Plans a commit from the current assignment snapshot and spawns the blocking Sync commit
    /// into the single in-flight slot. Emits the ledger and team gauges even when there is nothing
    /// to commit.
    fn start_commit(&self, ledger: &mut Ledger) -> Option<CommitFuture> {
        let assigned = match self.consumer.assigned_partitions() {
            Ok(partitions) => partitions.into_iter().map(SourcePartition).collect(),
            Err(err) => {
                warn!(error = %err, "failed to snapshot partition assignment; skipping commit tick");
                return None;
            }
        };
        let plan = ledger.commit_plan(&assigned);
        if !plan.pruned.is_empty() {
            debug!(pruned = ?plan.pruned, "pruned revoked partitions from ledger");
        }

        let stats = ledger.stats();
        gauge!(FORWARDS_INFLIGHT).set(stats.in_flight as f64);
        gauge!(UNCOMMITTED_EVENTS).set(stats.uncommitted_events as f64);
        gauge!(LEDGER_PARTITIONS).set(stats.partitions as f64);
        gauge!(ACTIVE_TEAMS).set(self.team_gate.passing_teams() as f64);

        if plan.offsets.is_empty() {
            return None;
        }
        let raw: Vec<(i32, i64)> = plan.offsets.iter().map(|&(p, n)| (p.0, n.0)).collect();
        let consumer = self.consumer.clone();
        let task = tokio::task::spawn_blocking(move || consumer.commit_partition_offsets(&raw));
        Some(Box::pin(async move { (plan.offsets, task.await) }))
    }

    /// Sequential shutdown: settle the in-flight commit, flush the producer, drain remaining
    /// acks, then one final Sync commit. Every step runs under one shared
    /// [`SHUTDOWN_DRAIN_BUDGET`] deadline with the final commit's slice reserved, so the drain
    /// provably fits the 15s graceful window. Anything unresolved at a deadline simply replays
    /// under the new consumer (at-least-once).
    async fn drain_and_commit(
        &self,
        mut ledger: Ledger,
        mut pending_acks: FuturesUnordered<AckFuture>,
        mut pending_commit: FuturesUnordered<CommitFuture>,
        intake: Intake,
    ) {
        let deadline = tokio::time::Instant::now() + SHUTDOWN_DRAIN_BUDGET;
        let pre_commit_deadline = deadline - SHUTDOWN_COMMIT_RESERVE;

        // Settle the in-flight commit first so the final plan is a real delta.
        if !pending_commit.is_empty() {
            match tokio::time::timeout_at(pre_commit_deadline, pending_commit.next()).await {
                Ok(Some((offsets, result))) => {
                    apply_commit_result(&mut ledger, offsets, result);
                }
                Ok(None) => {}
                Err(_) => warn!("in-flight commit did not settle before shutdown deadline"),
            }
        }

        // A forward parked on QueueFull gets one last attempt; if the queue is still full its
        // offset was never observed, so the commit stops short of it and the new owner replays.
        if let Intake::RetryForward {
            forward, backoff, ..
        } = intake
        {
            if let Intake::RetryForward { forward, .. } =
                self.try_enqueue(forward, backoff, &mut ledger, &mut pending_acks)
            {
                warn!(
                    partition = forward.partition.0,
                    offset = forward.offset.0,
                    "producer queue still full at shutdown; parked forward will replay"
                );
            }
        }

        let producer = self.producer.clone();
        let flush = tokio::task::spawn_blocking(move || producer.flush(SHUTDOWN_FLUSH_TIMEOUT));
        match tokio::time::timeout_at(pre_commit_deadline, flush).await {
            Ok(Ok(Ok(()))) => {}
            Ok(Ok(Err(err))) => warn!(error = %err, "producer flush failed during shutdown"),
            Ok(Err(err)) => error!(error = %err, "producer flush task panicked"),
            Err(_) => warn!("producer flush exceeded the shutdown budget"),
        }

        while !pending_acks.is_empty() {
            match tokio::time::timeout_at(pre_commit_deadline, pending_acks.next()).await {
                Ok(Some(ack)) => {
                    record_delivery(&mut ledger, ack);
                }
                Ok(None) => break,
                Err(_) => {
                    warn!(
                        unresolved = pending_acks.len(),
                        "ack drain deadline reached; unresolved forwards will replay"
                    );
                    break;
                }
            }
        }

        if let Some(commit) = self.start_commit(&mut ledger) {
            match tokio::time::timeout_at(deadline, commit).await {
                Ok((offsets, result)) => {
                    apply_commit_result(&mut ledger, offsets, result);
                }
                Err(_) => warn!("final offset commit timed out; committed work will replay"),
            }
        }
    }
}

fn record_delivery(ledger: &mut Ledger, ack: DeliveryAck) {
    let DeliveryAck {
        partition,
        offset,
        sent_at,
        result,
    } = ack;
    let outcome = match &result {
        Ok(()) => {
            counter!(EVENTS_FORWARDED).increment(1);
            DeliveryOutcome::Acked
        }
        Err(err) => {
            counter!(PRODUCE_ERRORS).increment(1);
            counter!(EVENTS_ABANDONED).increment(1);
            // Counter-visible rather than log-visible: a broker outage abandons at consume rate.
            debug!(
                error = %err,
                partition = partition.0,
                offset = offset.0,
                "delivery failed after librdkafka retries; abandoning event"
            );
            DeliveryOutcome::Abandoned
        }
    };
    let label = match outcome {
        DeliveryOutcome::Acked => "acked",
        DeliveryOutcome::Abandoned => "abandoned",
    };
    histogram!(PRODUCE_ACK_SECONDS, "outcome" => label).record(sent_at.elapsed().as_secs_f64());

    if ledger.resolve(partition, offset, outcome) == Resolution::Untracked {
        debug!(
            partition = partition.0,
            offset = offset.0,
            "straggler delivery for a pruned partition"
        );
    }
}

/// Returns true when the broker confirmed the commit (and the ledger was updated). Failed
/// commits are only counted: the unconfirmed delta is re-emitted at the next tick.
fn apply_commit_result(
    ledger: &mut Ledger,
    offsets: Vec<(SourcePartition, NextOffset)>,
    result: CommitResult,
) -> bool {
    match result {
        Ok(Ok(())) => {
            ledger.confirm_committed(&offsets);
            counter!(COMMITS).increment(1);
            true
        }
        Ok(Err(err)) => {
            counter!(COMMIT_ERRORS).increment(1);
            warn!(error = %err, "offset commit failed; retrying next tick");
            false
        }
        Err(err) => {
            counter!(COMMIT_ERRORS).increment(1);
            error!(error = %err, "offset commit task panicked");
            false
        }
    }
}

/// Liveness gate for the pipeline loop. Stale-and-committable is the only state that withholds the
/// heartbeat — a committer wedged for [`COMMIT_STALENESS_FACTOR`] intervals while committable work
/// waits must trip the stall detector. An idle topic (nothing committable) stays healthy even when
/// the last commit is old, and any fresh commit clears the staleness.
fn should_heartbeat(commit_stale: bool, has_committable: bool) -> bool {
    !commit_stale || !has_committable
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::Arc;

    use crate::event::{sample_clickhouse_event, RawJsonString};
    use crate::filter_team_index::TeamIndex;
    use common_cohort_team_access::TeamAccess;
    use common_types::cohort::TeamAllowlist;
    use serde_json::{json, Value};

    /// Every team processes, so only the index decides.
    fn gate_with(teams: &[i32]) -> TeamGate {
        TeamGate::new(
            Arc::new(TeamIndex::from_teams(teams.iter().copied())),
            TeamAccess::with_granted(TeamAllowlist::All, []),
        )
    }

    fn gate_fields(team_id: i32, person_id: Option<IgnoredAny>) -> GateFields {
        GateFields { team_id, person_id }
    }

    #[test]
    fn heartbeat_withheld_only_when_stale_and_committable() {
        // The single false case: a wedged committer with work waiting must stop heartbeating so
        // the stall detector restarts the pod.
        assert!(!should_heartbeat(true, true));
        // Stale but nothing to commit = idle topic, stays healthy.
        assert!(should_heartbeat(true, false));
        // Fresh commits keep the pod healthy regardless of pending work.
        assert!(should_heartbeat(false, true));
        assert!(should_heartbeat(false, false));
    }

    #[test]
    fn classify_gate_covers_every_gate_outcome() {
        let cases = [
            (
                "forward",
                Some(IgnoredAny),
                2,
                vec![2],
                GateOutcome::Forward,
            ),
            ("no person", None, 2, vec![2], GateOutcome::DropNoPersonId),
            (
                "team gate",
                Some(IgnoredAny),
                99,
                vec![2],
                GateOutcome::SkipTeamGate,
            ),
            (
                "no person precedence",
                None,
                99,
                vec![2],
                GateOutcome::DropNoPersonId,
            ),
            (
                "empty index",
                Some(IgnoredAny),
                2,
                vec![],
                GateOutcome::SkipTeamGate,
            ),
        ];

        for (name, person_id, team_id, teams, expected) in cases {
            let gate = gate_with(&teams);
            assert_eq!(
                classify_gate(&gate_fields(team_id, person_id), &gate),
                expected,
                "case: {name}"
            );
        }
    }

    #[test]
    fn unloaded_index_forwards_nothing() {
        let gate = TeamGate::new(
            Arc::new(TeamIndex::new()),
            TeamAccess::with_granted(TeamAllowlist::All, []),
        );
        assert_eq!(
            classify_gate(&gate_fields(2, Some(IgnoredAny)), &gate),
            GateOutcome::SkipTeamGate
        );
    }

    #[test]
    fn dropped_event_never_materializes_blobs() {
        // A JSON-object `properties` fails the survivor parse (the field is a stringified blob):
        // outside the index the gate ignores the blob and skips; inside it the survivor parse rejects
        // it as unparseable.
        let mut value: Value =
            serde_json::to_value(sample_clickhouse_event(99, Some("p"))).unwrap();
        value["properties"] = json!({ "a": [1, 2, 3] });
        let payload = serde_json::to_vec(&value).unwrap();

        assert!(matches!(
            decode_gated(&payload, &gate_with(&[2])),
            Decoded::SkipTeamGate
        ));
        assert!(matches!(
            decode_gated(&payload, &gate_with(&[99])),
            Decoded::Unparseable
        ));
    }

    #[test]
    fn garbage_payload_is_unparseable_not_an_error() {
        assert!(matches!(
            decode_gated(b"not json at all", &gate_with(&[2])),
            Decoded::Unparseable
        ));
    }

    #[test]
    fn missing_person_mode_skips_mismatched_team_instead_of_erroring() {
        // `person_mode` is required by the survivor parse but ignored by the gate.
        let mut value: Value =
            serde_json::to_value(sample_clickhouse_event(99, Some("p"))).unwrap();
        value.as_object_mut().unwrap().remove("person_mode");
        let payload = serde_json::to_vec(&value).unwrap();

        assert!(matches!(
            decode_gated(&payload, &gate_with(&[2])),
            Decoded::SkipTeamGate
        ));
    }

    #[test]
    fn person_id_presence_matches_full_parse_semantics() {
        let gate = gate_with(&[2]);
        let base: Value =
            serde_json::to_value(sample_clickhouse_event(2, Some("ignored"))).unwrap();
        let decode = |value: Value| decode_gated(&serde_json::to_vec(&value).unwrap(), &gate);

        // Absent: drop (person check runs before the team gate).
        let mut absent = base.clone();
        absent.as_object_mut().unwrap().remove("person_id");
        assert!(matches!(decode(absent), Decoded::DropNoPersonId));

        // `null`: drop.
        let mut null = base.clone();
        null["person_id"] = Value::Null;
        assert!(matches!(decode(null), Decoded::DropNoPersonId));

        // Empty string is present: forward.
        let mut empty = base.clone();
        empty["person_id"] = json!("");
        match decode(empty) {
            Decoded::Forward(fields) => assert_eq!(fields.person_id, ""),
            other => panic!("expected Forward, got {other:?}"),
        }

        // Valid string: forward.
        let mut valid = base.clone();
        valid["person_id"] = json!("p");
        match decode(valid) {
            Decoded::Forward(fields) => assert_eq!(fields.person_id, "p"),
            other => panic!("expected Forward, got {other:?}"),
        }
    }

    #[test]
    fn survivor_parses_the_forwarded_fields() {
        let payload = serde_json::to_vec(&sample_clickhouse_event(2, Some("p"))).unwrap();
        match decode_gated(&payload, &gate_with(&[2])) {
            Decoded::Forward(fields) => {
                assert_eq!(fields.person_id, "p");
                assert_eq!(fields.team_id, 2);
                // Blobs skipped on the drop path are captured, still escaped, for survivors.
                assert_eq!(
                    fields.properties.as_ref().map(RawJsonString::get),
                    Some(r#""{\"$current_url\":\"/pricing\"}""#)
                );
                assert_eq!(
                    fields.person_properties.as_ref().map(RawJsonString::get),
                    Some(r#""{\"email\":\"[email protected]\"}""#)
                );
            }
            other => panic!("expected Forward, got {other:?}"),
        }
    }
}