Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.6 KB
Raw Blame
//! When accepted offsets reach the ledger and the commit pacer: per group.

mod budget;
#[cfg(test)]
mod fakes;
mod group;
mod uncommitted;

use std::collections::HashMap;
use std::time::Instant;

use common_kafka_consumer::{Charge, Held, Offset, Rejection, TopicPartition};

#[cfg(test)]
use common_kafka_consumer::TopicOffsetLedger;

use budget::PartitionBudget;
pub(crate) use group::GroupSettlement;
use uncommitted::Uncommitted;

pub(crate) trait Settlement: Send + Sync {
    fn charge(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        charges: &[(Offset, Charge)],
    ) -> Result<Charged, Rejection>;

    /// Accepted offsets from one poll's slice of a partition, charged under `generation`.
    fn settle_run(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        offsets: &[Offset],
    ) -> anyhow::Result<RunOutcome>;

    fn take_due(&self, now: Instant) -> Option<Commit>;

    fn take_all(&self) -> Option<Commit>;

    fn on_assigned(&self, assigned: &[TopicPartition]);

    fn on_revoked(&self, revoked: &[TopicPartition]);
}

pub(crate) trait Accounting: Send + Sync {
    fn charge(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        charges: &[(Offset, Charge)],
    ) -> Result<Held, Rejection>;
    fn settled(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        offsets: impl IntoIterator<Item = Offset>,
    ) -> Result<Option<Offset>, Rejection>;
    fn take_due(&self, now: Instant) -> Option<HashMap<TopicPartition, HandedOut>>;
    fn take_all(&self) -> Option<HashMap<TopicPartition, HandedOut>>;
    fn forget_partitions(&self, topic_partitions: &[TopicPartition]);
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct HandedOut {
    pub(crate) offset: Offset,
    pub(crate) remaining: Held,
}

pub(crate) trait Admission: Send + Sync {
    fn charged(&self, topic_partition: &TopicPartition, uncommitted: Charge) -> Option<Pause>;
    fn handed_out(&self, handed_out: &HashMap<TopicPartition, HandedOut>) -> Vec<TopicPartition>;

    fn forget_partitions(&self, topic_partitions: &[TopicPartition]);
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RunOutcome {
    Settled,
    /// Charged under a generation that a rebalance has since replaced.
    Stale,
}

#[derive(Debug, PartialEq, Eq)]
pub(crate) struct Pause;

#[derive(Debug, PartialEq, Eq)]
pub(crate) struct Charged {
    pub(crate) uncommitted: Charge,
    pub(crate) pause: Option<Pause>,
}

pub(crate) struct Commit {
    pub(crate) offsets: HashMap<TopicPartition, Offset>,
    pub(crate) uncommitted: HashMap<TopicPartition, Charge>,
    pub(crate) resume: Vec<TopicPartition>,
}

impl Commit {
    fn new(handed_out: HashMap<TopicPartition, HandedOut>, resume: Vec<TopicPartition>) -> Self {
        let mut offsets = HashMap::with_capacity(handed_out.len());
        let mut uncommitted = HashMap::with_capacity(handed_out.len());
        for (topic_partition, handed_out) in handed_out {
            uncommitted.insert(topic_partition.clone(), handed_out.remaining.charge);
            offsets.insert(topic_partition, handed_out.offset);
        }
        Self {
            offsets,
            uncommitted,
            resume,
        }
    }
}

#[cfg(test)]
pub(crate) fn charge_offsets(
    ledger: &TopicOffsetLedger,
    topic_partition: &TopicPartition,
    generation: u64,
    offsets: &[i64],
) {
    ledger
        .charge(
            topic_partition,
            generation,
            offsets.iter().map(|&o| (Offset(o), Charge::ZERO)),
        )
        .expect("the test ledger accepts the charge");
}