Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

15.8 KB
Raw Blame
//! Where each input topic resumes, and who committed it, read from its consumer group on the
//! broker. A checkpoint records these positions, and a restore commits them back.
//!
//! Every commit the processor makes follows a WAL fsync, so a group's committed offset is at or
//! below the durable state, and a checkpoint taken after reading it contains at least that state.
//! That holds only for this pod's own commits, so a checkpoint takes no position another split
//! stamped (see [`super::fence`]).

use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::time::Duration;

use rdkafka::consumer::{BaseConsumer, CommitMode, Consumer, ConsumerContext};
use rdkafka::error::{KafkaError, KafkaResult};
use rdkafka::types::RDKafkaErrorCode;
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
use serde::{Deserialize, Serialize};

use super::fence::{ClaimedSplit, CommitLedger, Committed, InputCommits};
use super::stamp::{CommitStamp, Stamp};
use super::PartitionOwnership;
use crate::config::Config;

const FETCH_TIMEOUT: Duration = Duration::from_secs(10);

/// An input topic's name. A newtype, so a topic and a consumer group name cannot be swapped.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct InputTopic(String);

impl InputTopic {
    pub fn new(topic: impl Into<String>) -> Self {
        Self(topic.into())
    }

    pub fn as_str(&self) -> &str {
        &self.0
    }
}

impl fmt::Display for InputTopic {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.0)
    }
}

/// The next offset a consumer reads on one input partition. Never negative.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(try_from = "i64", into = "i64")]
pub struct ResumeOffset(i64);

#[derive(Debug, thiserror::Error)]
#[error("offset {0} is negative")]
pub struct NegativeOffset(i64);

impl TryFrom<i64> for ResumeOffset {
    type Error = NegativeOffset;

    fn try_from(offset: i64) -> Result<Self, Self::Error> {
        if offset < 0 {
            return Err(NegativeOffset(offset));
        }
        Ok(Self(offset))
    }
}

impl From<ResumeOffset> for i64 {
    fn from(offset: ResumeOffset) -> Self {
        offset.0
    }
}

impl ResumeOffset {
    pub fn get(self) -> i64 {
        self.0
    }
}

/// The `[low, high]` watermarks of `partitions` on `topic`. Asks each partition leader once per
/// bound, where `Consumer::fetch_watermarks` asks twice per partition: a restore checks every
/// partition of every input, and the boot waits on it.
fn read_watermarks<C, X>(
    client: &C,
    topic: &InputTopic,
    partitions: impl IntoIterator<Item = u16>,
) -> KafkaResult<BTreeMap<u16, (i64, i64)>>
where
    C: Consumer<X>,
    X: ConsumerContext,
{
    let partitions: Vec<(&InputTopic, u16)> = partitions
        .into_iter()
        .map(|partition| (topic, partition))
        .collect();
    let low = list_offsets(client, &partitions, Offset::Beginning)?;
    let high = list_offsets(client, &partitions, Offset::End)?;
    Ok(partitions
        .into_iter()
        .map(|(_, partition)| partition)
        .zip(low.into_iter().zip(high))
        .collect())
}

/// Where `at` falls on each of `partitions`, in their order, from one `ListOffsets` request per
/// partition leader, all within one timeout. `ListOffsets` reads the special timestamps `Beginning`
/// and `End` as the low and high watermarks, exactly as `fetch_watermarks` asks for them.
pub fn list_offsets<C, X>(
    client: &C,
    partitions: &[(&InputTopic, u16)],
    at: Offset,
) -> KafkaResult<Vec<i64>>
where
    C: Consumer<X>,
    X: ConsumerContext,
{
    // librdkafka fails `offsets_for_times` on an empty list with `InvalidArg`.
    if partitions.is_empty() {
        return Ok(Vec::new());
    }
    let mut request = TopicPartitionList::new();
    for &(topic, partition) in partitions {
        request.add_partition_offset(topic.as_str(), i32::from(partition), at)?;
    }
    let answered = client.offsets_for_times(request, FETCH_TIMEOUT)?;
    partitions
        .iter()
        .map(|&(topic, partition)| {
            let elem = answered
                .find_partition(topic.as_str(), i32::from(partition))
                .ok_or(KafkaError::OffsetFetch(RDKafkaErrorCode::UnknownPartition))?;
            elem.error()?;
            match elem.offset() {
                Offset::Offset(offset) => Ok(offset),
                _ => Err(KafkaError::OffsetFetch(RDKafkaErrorCode::InvalidArgument)),
            }
        })
        .collect()
}

/// One input's resume positions, read from its consumer group.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct InputPositions {
    topic: InputTopic,
    offsets: BTreeMap<u16, ResumeOffset>,
}

impl InputPositions {
    pub fn new(topic: InputTopic, offsets: BTreeMap<u16, ResumeOffset>) -> Self {
        Self { topic, offsets }
    }

    pub fn topic(&self) -> &InputTopic {
        &self.topic
    }

    pub fn offsets(&self) -> &BTreeMap<u16, ResumeOffset> {
        &self.offsets
    }
}

#[derive(Debug, thiserror::Error)]
pub enum PositionsError {
    #[error(transparent)]
    Kafka(#[from] KafkaError),
    #[error(transparent)]
    Negative(#[from] NegativeOffset),
    #[error("{topic} partition {partition} was last committed by {stamp}, not this pod")]
    Foreign {
        topic: InputTopic,
        partition: u16,
        stamp: Stamp,
    },
}

/// Every input's resume positions on partitions of a claimed split, none of them stamped by
/// another split. Only [`InputGroups::owned_positions`] builds one, so a manifest never records a
/// position another split stamped. An unstamped commit, such as an offset reset, passes.
#[derive(Debug)]
pub struct OwnedPositions {
    split: PartitionOwnership,
    inputs: Vec<InputPositions>,
}

impl OwnedPositions {
    #[cfg(test)]
    pub(crate) fn for_test(split: PartitionOwnership, inputs: Vec<InputPositions>) -> Self {
        Self { split, inputs }
    }

    pub fn split(&self) -> PartitionOwnership {
        self.split
    }

    pub fn inputs(&self) -> &[InputPositions] {
        &self.inputs
    }
}

/// Reads and commits one input's consumer-group positions without joining the group. Every method
/// blocks, so async callers go through `spawn_blocking`.
struct GroupReader {
    topic: InputTopic,
    config: ClientConfig,
}

impl GroupReader {
    fn new(config: &Config, topic: &str, group: &str) -> Self {
        Self {
            topic: InputTopic::new(topic),
            config: config.follower_client_config(group),
        }
    }

    /// A client that never subscribes or assigns, so it outlives the group's own consumer at
    /// shutdown and never takes part in a rebalance. Each call builds its own and drops it: nothing
    /// polls these clients, so one kept between calls would hold every log and error event
    /// librdkafka queues for it.
    fn client(&self) -> KafkaResult<BaseConsumer> {
        self.config.create()
    }

    fn topic(&self) -> &InputTopic {
        &self.topic
    }

    /// The group's last commit on each of `partitions`, from one `OffsetFetch`.
    fn commits(
        &self,
        client: &BaseConsumer,
        partitions: &BTreeSet<u16>,
    ) -> Result<InputCommits, PositionsError> {
        let topic = self.topic.as_str();
        let mut request = TopicPartitionList::new();
        for &partition in partitions {
            request.add_partition(topic, i32::from(partition));
        }
        let answered = client.committed_offsets(request, FETCH_TIMEOUT)?;
        let mut commits = BTreeMap::new();
        for &partition in partitions {
            let committed = match answered.find_partition(topic, i32::from(partition)) {
                Some(elem) => {
                    elem.error()?;
                    match elem.offset() {
                        Offset::Offset(next) => Committed::At {
                            offset: ResumeOffset::try_from(next)?,
                            stamp: Stamp::parse(elem.metadata()),
                        },
                        _ => Committed::Never,
                    }
                }
                None => Committed::Never,
            };
            commits.insert(partition, committed);
        }
        Ok(InputCommits::new(self.topic.clone(), commits))
    }

    /// Where the group resumes on each partition of `commits`: its committed offset, or the low
    /// watermark where it has none. Every consumer here starts at `earliest`, so replay from the low
    /// watermark misses nothing.
    fn resume_positions(
        &self,
        client: &BaseConsumer,
        commits: &InputCommits,
    ) -> Result<BTreeMap<u16, ResumeOffset>, PositionsError> {
        let mut positions = BTreeMap::new();
        let mut uncommitted = Vec::new();
        for (&partition, committed) in commits.commits() {
            match committed {
                Committed::At { offset, .. } => {
                    positions.insert(partition, *offset);
                }
                Committed::Never => uncommitted.push((&self.topic, partition)),
            }
        }
        let lows = list_offsets(client, &uncommitted, Offset::Beginning)?;
        for (&(_, partition), low) in uncommitted.iter().zip(lows) {
            positions.insert(partition, ResumeOffset::try_from(low)?);
        }
        Ok(positions)
    }

    /// Refuses a position another split stamped.
    fn owned_positions(
        &self,
        owner: &PartitionOwnership,
        partitions: &BTreeSet<u16>,
    ) -> Result<InputPositions, PositionsError> {
        let client = self.client()?;
        let commits = self.commits(&client, partitions)?;
        for (&partition, committed) in commits.commits() {
            if let Some(stamp) = committed.stamp().filter(|stamp| stamp.is_foreign_to(owner)) {
                return Err(PositionsError::Foreign {
                    topic: self.topic.clone(),
                    partition,
                    stamp: stamp.clone(),
                });
            }
        }
        let positions = self.resume_positions(&client, &commits)?;
        Ok(InputPositions::new(self.topic.clone(), positions))
    }

    /// Commits each partition of `commits` again at its resume position, with `stamp`.
    fn restamp(&self, stamp: &CommitStamp, commits: &InputCommits) -> Result<(), PositionsError> {
        let positions = self.resume_positions(&self.client()?, commits)?;
        Ok(self.commit(stamp, &positions)?)
    }

    fn commit(
        &self,
        stamp: &CommitStamp,
        positions: &BTreeMap<u16, ResumeOffset>,
    ) -> KafkaResult<()> {
        if positions.is_empty() {
            return Ok(());
        }
        let offsets = positions
            .iter()
            .map(|(&partition, &offset)| (i32::from(partition), offset.get()));
        stamp
            .list(self.topic.as_str(), offsets)?
            .commit(&self.client()?, CommitMode::Sync)
    }

    fn watermarks(
        &self,
        partitions: impl IntoIterator<Item = u16>,
    ) -> KafkaResult<BTreeMap<u16, (i64, i64)>> {
        read_watermarks(&self.client()?, &self.topic, partitions)
    }
}

/// A follower's group has no members, so a commit from any client needs no generation. The
/// restore commits follower positions through it before any consumer runs.
pub struct FollowerGroup(GroupReader);

impl FollowerGroup {
    pub fn topic(&self) -> &InputTopic {
        self.0.topic()
    }

    pub fn commit(
        &self,
        stamp: &CommitStamp,
        positions: &BTreeMap<u16, ResumeOffset>,
    ) -> KafkaResult<()> {
        self.0.commit(stamp, positions)
    }
}

/// Every enabled input, built once at startup and shared by the restore, the checkpoint sweeper and
/// the final checkpoint.
pub struct InputGroups {
    events: GroupReader,
    followers: Vec<FollowerGroup>,
    inputs: BTreeSet<InputTopic>,
}

impl InputGroups {
    pub fn new(config: &Config) -> Self {
        let events = GroupReader::new(
            config,
            &config.cohort_stream_events_topic,
            &config.kafka_consumer_group,
        );
        let mut follower_inputs = vec![
            (
                &config.person_merge_events_topic,
                &config.kafka_merge_consumer_group,
            ),
            (
                &config.cohort_merge_state_transfer_topic,
                &config.kafka_merge_apply_consumer_group,
            ),
        ];
        if config.cohort_cascade_enabled {
            follower_inputs.push((
                &config.cohort_cascade_events_topic,
                &config.kafka_cascade_consumer_group,
            ));
        }
        if config.cohort_seed_consumer_enabled {
            follower_inputs.push((
                &config.cohort_stream_seed_events_topic,
                &config.kafka_seed_consumer_group,
            ));
        }
        let followers = follower_inputs
            .into_iter()
            .map(|(topic, group)| FollowerGroup(GroupReader::new(config, topic, group)))
            .collect::<Vec<_>>();

        let inputs = std::iter::once(events.topic().clone())
            .chain(followers.iter().map(|follower| follower.topic().clone()))
            .collect();
        Self {
            events,
            followers,
            inputs,
        }
    }

    pub fn inputs(&self) -> &BTreeSet<InputTopic> {
        &self.inputs
    }

    pub fn followers(&self) -> &[FollowerGroup] {
        &self.followers
    }

    /// The last commit on `partitions`, on every enabled group.
    pub fn ledger(&self, partitions: &BTreeSet<u16>) -> Result<CommitLedger, PositionsError> {
        let read = |reader: &GroupReader| reader.commits(&reader.client()?, partitions);
        let events = read(&self.events)?;
        let followers = self
            .followers
            .iter()
            .map(|follower| read(&follower.0))
            .collect::<Result<_, _>>()?;
        Ok(CommitLedger::new(events, followers))
    }

    /// Commits the partitions of `ledger`, read by [`Self::ledger`], again with `stamp` on every
    /// enabled group.
    pub(super) fn restamp(
        &self,
        stamp: &CommitStamp,
        ledger: &CommitLedger,
    ) -> Result<(), PositionsError> {
        self.readers()
            .zip(ledger.inputs())
            .try_for_each(|(reader, commits)| {
                debug_assert_eq!(reader.topic(), commits.topic());
                reader.restamp(stamp, commits)
            })
    }

    /// Every input's resume positions on `partitions`, as long as no other split stamped them.
    pub fn owned_positions(
        &self,
        claimed: &ClaimedSplit,
        partitions: &BTreeSet<u16>,
    ) -> Result<OwnedPositions, PositionsError> {
        let split = claimed.ownership();
        let inputs = self
            .readers()
            .map(|reader| reader.owned_positions(&split, partitions))
            .collect::<Result<_, _>>()?;
        Ok(OwnedPositions { split, inputs })
    }

    /// `topic`'s watermarks on partitions `0..partition_count`.
    pub fn watermarks(
        &self,
        topic: &InputTopic,
        partition_count: u16,
    ) -> KafkaResult<BTreeMap<u16, (i64, i64)>> {
        self.readers()
            .find(|reader| reader.topic() == topic)
            .ok_or(KafkaError::MetadataFetch(
                RDKafkaErrorCode::UnknownTopicOrPartition,
            ))?
            .watermarks(0..partition_count)
    }

    fn readers(&self) -> impl Iterator<Item = &GroupReader> {
        std::iter::once(&self.events).chain(self.followers.iter().map(|follower| &follower.0))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rdkafka::ClientConfig;

    #[test]
    fn an_empty_request_reads_nothing_from_the_broker() {
        let unreachable: BaseConsumer = ClientConfig::new()
            .set("bootstrap.servers", "127.0.0.1:1")
            .create()
            .unwrap();
        assert_eq!(
            list_offsets(&unreachable, &[], Offset::Beginning).unwrap(),
            Vec::<i64>::new()
        );
    }
}