Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.8 KB
Raw Blame
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;

use common_kafka_consumer::{Charge, GroupCompletion, Partition, TopicPartition};
use metrics::{counter, gauge};
use rdkafka::message::Message;
use tokio::time::Instant;

use super::poll::InFlightPoll;
use super::{report_revoked, Collector, Parts, PollingStrategy, Step};
use crate::charge_limit::ChargeLimit;
use crate::settlement::{Commit, GroupSettlement, RunOutcome, Settlement};

/// Nothing waits on a submitted poll: the partition budget and the in-flight
/// limit bound admission.
pub(crate) struct ContinuousPolling {
    collector: Collector,
    settlement: Arc<GroupSettlement>,
    uncommitted: UncommittedCharges,
    processed: ProcessedTimestamps,
    limits: ChargeLimit,
    at_limit: bool,
    accepting_new_batches: bool,
    commit_due: bool,
}

impl ContinuousPolling {
    pub(crate) fn new(parts: Parts, limits: ChargeLimit) -> Self {
        Self {
            collector: Collector::new(parts.limits),
            settlement: parts.settlement,
            uncommitted: UncommittedCharges::default(),
            processed: ProcessedTimestamps::new(parts.group_id),
            limits,
            at_limit: false,
            accepting_new_batches: true,
            commit_due: false,
        }
    }
}

impl PollingStrategy for ContinuousPolling {
    /// The poll in progress is not counted, so the uncommitted charge can
    /// pass the limit by one poll.
    fn reading(&mut self) -> bool {
        let at_limit = self.limits.reached(self.uncommitted.total());
        if at_limit && !self.at_limit {
            counter!("ingestion_consumer_in_flight_limit_reached_total").increment(1);
        }
        self.at_limit = at_limit;
        self.accepting_new_batches && !at_limit
    }

    fn deadline(&self) -> Option<Instant> {
        self.collector.deadline()
    }

    fn on_received<M: Message>(&mut self, message: &M, now: Instant) {
        self.collector.add(message, now);
    }

    fn on_completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {
        if (completion.accepted as usize) < completion.offsets.len() {
            anyhow::bail!(
                "accepted {}/{} messages — not committing offsets",
                completion.accepted,
                completion.offsets.len()
            );
        }
        let Some(topic_partition) = self.uncommitted.topic_partition(completion.partition) else {
            return Ok(());
        };
        let outcome = self.settlement.settle_run(
            topic_partition,
            completion.generation,
            &completion.offsets,
        )?;
        if outcome == RunOutcome::Stale {
            return Ok(());
        }
        counter!("ingestion_consumer_messages_processed_total")
            .increment(completion.offsets.len() as u64);
        self.commit_due = true;
        Ok(())
    }

    fn on_ticked(&mut self) {
        self.commit_due = true;
    }

    fn on_assigned(&mut self, topic_partitions: &[TopicPartition]) {
        self.forget_partitions(topic_partitions);
    }

    fn on_drain_started(&mut self, topic_partitions: &[TopicPartition]) {
        report_revoked(self.collector.remove_partitions(topic_partitions) as u64);
    }

    fn on_drain_finished(&mut self, topic_partitions: &[TopicPartition]) -> Option<Commit> {
        let commit = self
            .settlement
            .take_all()
            .map(|commit| self.record_commit(commit));
        self.on_revoked(topic_partitions);
        commit
    }

    fn on_revoked(&mut self, topic_partitions: &[TopicPartition]) {
        self.forget_partitions(topic_partitions);
    }

    fn on_shutdown(&mut self) {
        self.accepting_new_batches = false;
        self.commit_due = true;
        self.collector.discard();
    }

    fn on_submitted(&mut self, poll: InFlightPoll) {
        self.uncommitted.record_poll(&poll);
        self.processed.record_poll(&poll);
    }

    fn next_step(&mut self, now: Instant) -> Option<Step> {
        if std::mem::take(&mut self.commit_due) {
            let commit = if self.accepting_new_batches {
                self.settlement.take_due(now.into_std())
            } else {
                self.settlement.take_all()
            };
            if let Some(commit) = commit {
                return Some(Step::Commit(self.record_commit(commit)));
            }
        }
        if let Some(collected) = self.collector.take_ready(now) {
            return Some(Step::Submit(collected));
        }
        (!self.accepting_new_batches && self.uncommitted.is_empty()).then_some(Step::Stop)
    }
}

impl ContinuousPolling {
    fn record_commit(&mut self, commit: Commit) -> Commit {
        self.uncommitted.record_commit(&commit);
        self.processed.record_commit(&commit);
        commit
    }

    fn forget_partitions(&mut self, topic_partitions: &[TopicPartition]) {
        let building = self.collector.remove_partitions(topic_partitions) as u64;
        self.uncommitted.forget(topic_partitions);
        self.processed.forget(topic_partitions);
        report_revoked(building);
    }
}

struct ProcessedTimestamps {
    group_id: String,
    partitions: HashMap<TopicPartition, VecDeque<SubmittedSlice>>,
}

struct SubmittedSlice {
    last_offset: i64,
    latest_kafka_ts: i64,
}

impl ProcessedTimestamps {
    fn new(group_id: String) -> Self {
        Self {
            group_id,
            partitions: HashMap::new(),
        }
    }

    fn record_poll(&mut self, poll: &InFlightPoll) {
        for (topic_partition, deliveries) in &poll.partitions {
            self.partitions
                .entry(topic_partition.clone())
                .or_default()
                .push_back(SubmittedSlice {
                    last_offset: deliveries.span.last,
                    latest_kafka_ts: deliveries.latest_kafka_ts,
                });
        }
    }

    fn record_commit(&mut self, commit: &Commit) {
        for (topic_partition, next_to_read) in &commit.offsets {
            let Some(polls) = self.partitions.get_mut(topic_partition) else {
                continue;
            };
            let mut latest = None;
            while polls
                .front()
                .is_some_and(|slice| slice.last_offset < next_to_read.0)
            {
                let slice = polls.pop_front().expect("checked above");
                latest = latest.max(Some(slice.latest_kafka_ts));
            }
            if let Some(latest) = latest {
                // Same name as the Node.js consumer's gauge, so one dashboard reads both.
                gauge!(
                    "latest_processed_timestamp_ms",
                    "topic" => topic_partition.topic.clone(),
                    "partition" => topic_partition.partition.to_string(),
                    "groupId" => self.group_id.clone()
                )
                .set(latest as f64);
            }
        }
    }

    fn forget(&mut self, topic_partitions: &[TopicPartition]) {
        for topic_partition in topic_partitions {
            self.partitions.remove(topic_partition);
        }
    }
}

/// Keyed by partition number, because a completion carries no topic.
#[derive(Default)]
struct UncommittedCharges {
    partitions: HashMap<Partition, (TopicPartition, Charge)>,
    total: Charge,
}

impl UncommittedCharges {
    fn total(&self) -> Charge {
        self.total
    }

    fn is_empty(&self) -> bool {
        self.total == Charge::ZERO
    }

    /// `None` once the partition is forgotten, until its next charge.
    fn topic_partition(&self, partition: Partition) -> Option<&TopicPartition> {
        self.partitions
            .get(&partition)
            .map(|(topic_partition, _)| topic_partition)
    }

    fn record_poll(&mut self, poll: &InFlightPoll) {
        for (topic_partition, &charge) in &poll.uncommitted {
            let previous = self.partitions.insert(
                Partition(topic_partition.partition),
                (topic_partition.clone(), charge),
            );
            self.replace(previous.map(|(_, charge)| charge), charge);
        }
    }

    fn record_commit(&mut self, commit: &Commit) {
        for (topic_partition, &charge) in &commit.uncommitted {
            if let Some((_, partition)) = self
                .partitions
                .get_mut(&Partition(topic_partition.partition))
            {
                let previous = std::mem::replace(partition, charge);
                self.replace(Some(previous), charge);
            }
        }
    }

    fn forget(&mut self, topic_partitions: &[TopicPartition]) {
        for topic_partition in topic_partitions {
            let previous = self
                .partitions
                .remove(&Partition(topic_partition.partition));
            self.replace(previous.map(|(_, charge)| charge), Charge::ZERO);
        }
    }

    fn replace(&mut self, previous: Option<Charge>, charge: Charge) {
        self.total = self.total - previous.unwrap_or(Charge::ZERO) + charge;
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;
    use std::num::NonZeroU64;

    use common_kafka_consumer::Offset;
    use metrics_util::debugging::{DebugValue, DebuggingRecorder};

    use super::super::harness::{completion, p, Harness};
    use super::*;

    fn continuous(in_flight_events: u64) -> Harness<ContinuousPolling> {
        Harness::new(|parts| {
            ContinuousPolling::new(
                parts,
                ChargeLimit::new(NonZeroU64::new(in_flight_events), None),
            )
        })
    }

    #[test]
    fn the_continuous_strategy_stops_reading_one_poll_past_its_limit_until_a_commit() {
        let mut harness = continuous(1);

        harness.receive(0, &[0, 1]);
        assert!(
            harness.poller.reading(),
            "the poll in progress is not counted"
        );
        harness.after_the_batch_timeout();
        assert_eq!(harness.submit(), HashMap::from([(p(0), (0, 1))]));
        assert!(!harness.poller.reading());

        harness.complete(0, &[0, 1]);
        assert!(
            !harness.poller.reading(),
            "a completion frees no room before its commit"
        );
        assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));
        assert!(harness.poller.reading());
    }

    #[test]
    fn a_revoke_removes_the_partitions_messages_received_before_it() {
        let mut harness = continuous(1_000);
        harness.receive(0, &[10, 11]);
        harness.receive(1, &[20]);

        harness.poller.on_revoked(&[p(0)]);
        harness.receive(0, &[10]);
        harness.after_the_batch_timeout();

        assert_eq!(
            harness.submit(),
            HashMap::from([(p(0), (10, 10)), (p(1), (20, 20))]),
            "the redelivery after the revoke stays"
        );
    }

    #[test]
    fn a_completion_from_before_a_revoke_settles_nothing_in_the_next_assignment() {
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let _guard = metrics::set_default_local_recorder(&recorder);
        let processed = || {
            snapshotter
                .snapshot()
                .into_vec()
                .into_iter()
                .find_map(|(key, _, _, value)| match value {
                    DebugValue::Counter(count)
                        if key.key().name() == "ingestion_consumer_messages_processed_total" =>
                    {
                        Some(count)
                    }
                    _ => None,
                })
                .unwrap_or_default()
        };
        let mut harness = continuous(1_000);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();
        let before = harness.ledger.generation(&p(0));

        harness.revoke(0);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();

        harness.complete_under(before, 0, &[0, 1]);
        harness.poller.on_ticked();
        assert!(
            harness.step().is_none(),
            "the old assignment's completion is stale"
        );
        assert_eq!(processed(), 0, "a stale completion processed nothing");
        harness.complete(0, &[0, 1]);
        assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));
        assert_eq!(processed(), 2);
    }

    #[test]
    fn a_started_drain_drops_the_poll_in_progress_and_settles_what_was_submitted() {
        let mut harness = continuous(1_000);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();
        harness.receive(0, &[2]);

        harness.poller.on_drain_started(&[p(0)]);
        harness.after_the_batch_timeout();
        assert!(
            harness.step().is_none(),
            "offset 2 was never charged, so the new owner reads it"
        );

        harness.complete(0, &[0, 1]);
        assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));
    }

    #[test]
    fn a_finished_drain_commits_up_to_the_first_unprocessed_offset() {
        let mut harness = continuous(1_000);
        harness.receive(0, &[0, 1, 2]);
        harness.after_the_batch_timeout();
        harness.submit();

        harness.poller.on_drain_started(&[p(0)]);
        harness.complete(0, &[0, 2]);

        assert_eq!(
            harness.finish_drain(0),
            HashMap::from([(p(0), Offset(1))]),
            "offset 1 came back unfinished"
        );
    }

    #[test]
    fn a_finished_drain_frees_the_partitions_charge_and_ignores_its_late_completions() {
        let mut harness = continuous(3);
        harness.receive(0, &[0, 1, 2]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert!(!harness.poller.reading());
        let drained_generation = harness.ledger.generation(&p(0));

        harness.poller.on_drain_started(&[p(0)]);
        assert_eq!(harness.finish_drain(0), HashMap::new());
        assert!(
            harness.poller.reading(),
            "a drained partition holds no charge"
        );

        harness.complete_under(drained_generation, 0, &[0, 1, 2]);
        harness.poller.on_ticked();
        assert!(
            harness.step().is_none(),
            "a late completion commits nothing"
        );
    }

    #[test]
    fn a_revoke_frees_the_partitions_charge() {
        let mut harness = continuous(3);
        harness.receive(0, &[0, 1, 2]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert!(!harness.poller.reading());

        harness.revoke(0);
        assert!(
            harness.poller.reading(),
            "a revoked partition holds no charge"
        );

        harness.receive(0, &[0, 1, 2]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert!(
            !harness.poller.reading(),
            "the new assignment's charge counts in full"
        );
    }

    #[test]
    fn the_limit_sums_each_partitions_uncommitted_charge() {
        let mut harness = continuous(5);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();
        harness.receive(0, &[2, 3]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert!(
            harness.poller.reading(),
            "a second submit replaces the partition's charge"
        );

        harness.receive(1, &[0]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert!(
            !harness.poller.reading(),
            "the partitions' charges add up to the limit"
        );

        harness.complete(0, &[0, 1]);
        assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));
        assert!(
            harness.poller.reading(),
            "a commit frees only the offsets it passed"
        );
    }

    #[test]
    fn a_completion_a_worker_did_not_accept_in_full_fails() {
        let mut harness = continuous(1_000);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();
        let mut short = completion(harness.ledger.generation(&p(0)), 0, &[0, 1]);
        short.accepted = 1;

        let error = harness
            .poller
            .on_completed(&short)
            .expect_err("a dropped message must not commit");
        assert!(error.to_string().contains("accepted 1/2"), "{error}");
    }

    #[test]
    fn a_commit_reports_the_newest_kafka_timestamp_it_passed() {
        let recorder = DebuggingRecorder::new();
        let snapshotter = recorder.snapshotter();
        let _guard = metrics::set_default_local_recorder(&recorder);
        let latest_processed = || {
            snapshotter
                .snapshot()
                .into_vec()
                .into_iter()
                .find_map(|(key, _, _, value)| match value {
                    DebugValue::Gauge(ts)
                        if key.key().name() == "latest_processed_timestamp_ms" =>
                    {
                        Some(ts.into_inner())
                    }
                    _ => None,
                })
        };
        let mut harness = continuous(1_000);
        harness.receive(0, &[0, 1]);
        harness.after_the_batch_timeout();
        harness.submit();
        assert_eq!(latest_processed(), None);

        harness.complete(0, &[0, 1]);
        harness.commit();

        assert_eq!(latest_processed(), Some(1_000.0));
    }

    #[test]
    fn shutdown_drops_the_poll_in_progress_and_stops_once_in_flight_work_drains() {
        let mut harness = continuous(1_000);
        harness.receive(0, &[0]);
        harness.after_the_batch_timeout();
        harness.submit();
        harness.receive(0, &[1]);

        harness.poller.on_shutdown();
        assert!(!harness.poller.reading());
        assert_eq!(harness.poller.deadline(), None);
        assert!(harness.step().is_none(), "offset 0 is still in flight");

        harness.complete(0, &[0]);
        assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(1))]));
        assert!(matches!(harness.step(), Some(Step::Stop)));
    }
}