Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.8 KB
Raw Blame
use std::sync::Arc;
use std::time::{Duration, Instant};

use common_kafka_consumer::{Charge, Offset, Rejection, TopicOffsetLedger, TopicPartition};

use super::{
    Accounting, Admission, Charged, Commit, PartitionBudget, RunOutcome, Settlement, Uncommitted,
};
use crate::commit_pacer::CommitPacer;
use crate::ledger_rejection::{report_rejection, RejectedSlice};
use crate::order_sentinel::OffsetSpan;
use crate::partition_budget::BudgetLimits;

pub(crate) struct GroupSettlement<A: Accounting = Uncommitted, D: Admission = PartitionBudget> {
    accounting: A,
    admission: D,
}

impl GroupSettlement {
    pub(crate) fn new(
        ledger: Arc<TopicOffsetLedger>,
        commit_interval: Duration,
        limits: BudgetLimits,
    ) -> Self {
        Self {
            accounting: Uncommitted::new(ledger, CommitPacer::every(commit_interval)),
            admission: PartitionBudget::new(limits),
        }
    }
}

impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettlement<A, D> {
    fn charge(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        charges: &[(Offset, Charge)],
    ) -> Result<Charged, Rejection> {
        let uncommitted = self
            .accounting
            .charge(topic_partition, generation, charges)?
            .charge;
        Ok(Charged {
            uncommitted,
            pause: self.admission.charged(topic_partition, uncommitted),
        })
    }

    fn settle_run(
        &self,
        topic_partition: &TopicPartition,
        generation: u64,
        offsets: &[Offset],
    ) -> anyhow::Result<RunOutcome> {
        let (Some(first), Some(last)) = (offsets.first(), offsets.last()) else {
            return Ok(RunOutcome::Settled);
        };
        if let Err(rejection) =
            self.accounting
                .settled(topic_partition, generation, offsets.iter().copied())
        {
            report_rejection(
                "settle",
                topic_partition,
                rejection,
                RejectedSlice::settled(&OffsetSpan {
                    first: first.0,
                    last: last.0,
                }),
            )?;
            return Ok(RunOutcome::Stale);
        }
        Ok(RunOutcome::Settled)
    }

    fn take_due(&self, now: Instant) -> Option<Commit> {
        let handed_out = self.accounting.take_due(now)?;
        let resume = self.admission.handed_out(&handed_out);
        Some(Commit::new(handed_out, resume))
    }

    fn take_all(&self) -> Option<Commit> {
        let handed_out = self.accounting.take_all()?;
        let resume = self.admission.handed_out(&handed_out);
        Some(Commit::new(handed_out, resume))
    }

    fn on_assigned(&self, assigned: &[TopicPartition]) {
        self.forget_partitions(assigned);
    }

    fn on_revoked(&self, revoked: &[TopicPartition]) {
        self.forget_partitions(revoked);
    }
}

impl<A: Accounting, D: Admission> GroupSettlement<A, D> {
    fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
        self.accounting.forget_partitions(topic_partitions);
        self.admission.forget_partitions(topic_partitions);
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use common_kafka_consumer::{Held, LedgerError};

    use super::super::fakes::{FakeAccounting, FakeAdmission, Settled};
    use super::super::{HandedOut, Pause};
    use super::*;

    fn p(partition: i32) -> TopicPartition {
        TopicPartition::new("events", partition)
    }

    fn events(events: u64) -> Charge {
        Charge { events, bytes: 0 }
    }

    fn uncommitted_events(events: u64) -> Held {
        Held {
            offsets: events as usize,
            charge: Charge { events, bytes: 0 },
        }
    }

    fn handed_out(offset: i64, remaining_events: u64) -> HandedOut {
        HandedOut {
            offset: Offset(offset),
            remaining: uncommitted_events(remaining_events),
        }
    }

    fn settlement() -> (
        GroupSettlement<FakeAccounting, FakeAdmission>,
        FakeAccounting,
        FakeAdmission,
    ) {
        let accounting = FakeAccounting::default();
        let admission = FakeAdmission::default();
        let settlement = GroupSettlement {
            accounting: accounting.clone(),
            admission: admission.clone(),
        };
        (settlement, accounting, admission)
    }

    #[test]
    fn a_charge_asks_the_admission_with_the_uncommitted_charge() {
        let (settlement, accounting, admission) = settlement();
        accounting.charge_returns(Ok(uncommitted_events(10)));
        admission.charged_returns(Some(Pause));

        assert_eq!(
            settlement
                .charge(&p(0), 0, &[])
                .map(|charged| charged.pause),
            Ok(Some(Pause))
        );
        assert_eq!(admission.charges_seen(), vec![(p(0), events(10))]);
    }

    #[test]
    fn a_rejected_charge_asks_the_admission_nothing() {
        let (settlement, accounting, admission) = settlement();
        let stale = Rejection::Stale {
            stamp: 0,
            generation: 1,
        };
        accounting.charge_returns(Err(stale));

        assert_eq!(
            settlement
                .charge(&p(0), 0, &[])
                .map(|charged| charged.pause),
            Err(stale)
        );
        assert!(admission.charges_seen().is_empty());
    }

    #[test]
    fn a_hand_out_commits_its_offsets_and_resumes_what_the_admission_decides() {
        let (settlement, accounting, admission) = settlement();
        let hand_out = HashMap::from([(p(0), handed_out(11, 3)), (p(1), handed_out(21, 0))]);
        accounting.take_hands_out(hand_out.clone());
        admission.handed_out_returns(vec![p(0)]);

        let commit = settlement.take_due(Instant::now()).expect("handed out");

        assert_eq!(
            commit.offsets,
            HashMap::from([(p(0), Offset(11)), (p(1), Offset(21))])
        );
        assert_eq!(commit.resume, vec![p(0)]);
        assert_eq!(admission.hand_outs_seen(), vec![hand_out]);
    }

    #[test]
    fn take_all_asks_the_admission_like_take_due() {
        let (settlement, accounting, admission) = settlement();
        accounting.take_hands_out(HashMap::from([(p(0), handed_out(11, 0))]));
        admission.handed_out_returns(vec![p(0)]);

        assert_eq!(
            settlement.take_all().expect("handed out").resume,
            vec![p(0)]
        );
    }

    #[test]
    fn a_run_reaches_the_accounting_under_its_generation() {
        let (settlement, accounting, _) = settlement();

        settlement
            .settle_run(&p(0), 3, &[Offset(10), Offset(11)])
            .unwrap();

        assert_eq!(
            accounting.settles_seen(),
            vec![Settled {
                topic_partition: p(0),
                generation: 3,
                offsets: vec![Offset(10), Offset(11)],
            }]
        );
    }

    #[test]
    fn an_empty_run_settles_nothing() {
        let (settlement, accounting, _) = settlement();

        settlement.settle_run(&p(0), 0, &[]).unwrap();

        assert!(accounting.settles_seen().is_empty());
    }

    struct RejectedRun {
        rejection: Rejection,
        expected_error: bool,
    }

    #[rstest::rstest]
    #[case::stale(RejectedRun {
        rejection: Rejection::Stale { stamp: 0, generation: 1 },
        expected_error: false,
    })]
    #[case::violation(RejectedRun {
        rejection: Rejection::Violation {
            error: LedgerError::CompletedTwice { offset: Offset(10) },
            stamp: 0,
            generation: 0,
            held: Held::default(),
        },
        expected_error: true,
    })]
    fn only_a_violating_run_fails(#[case] run: RejectedRun) {
        let (settlement, accounting, _) = settlement();
        accounting.settle_returns(Err(run.rejection));

        assert_eq!(
            settlement.settle_run(&p(0), 0, &[Offset(10)]).is_err(),
            run.expected_error
        );
    }

    #[test]
    fn a_revoke_reaches_both_the_accounting_and_the_admission() {
        let (settlement, accounting, admission) = settlement();

        settlement.on_revoked(&[p(0)]);

        assert_eq!(accounting.forgotten(), vec![p(0)]);
        assert_eq!(admission.forgotten(), vec![p(0)]);
    }
}

#[cfg(test)]
mod integration_tests {
    use std::collections::HashMap;
    use std::num::NonZeroU64;

    use super::super::{charge_offsets, Pause};
    use super::*;

    fn settlement() -> (Arc<TopicOffsetLedger>, GroupSettlement) {
        let ledger = Arc::new(TopicOffsetLedger::new());
        let settlement =
            GroupSettlement::new(Arc::clone(&ledger), Duration::ZERO, BudgetLimits::unset());
        (ledger, settlement)
    }

    #[test]
    fn a_partition_commits_only_once_its_completed_prefix_is_contiguous() {
        let (ledger, settlement) = settlement();
        let p0 = TopicPartition::new("events", 0);
        charge_offsets(&ledger, &p0, 0, &[10, 11, 12]);

        settlement.settle_run(&p0, 0, &[Offset(12)]).unwrap();
        assert!(
            settlement.take_due(Instant::now()).is_none(),
            "the window base is still incomplete"
        );

        settlement
            .settle_run(&p0, 0, &[Offset(10), Offset(11)])
            .unwrap();
        assert_eq!(
            settlement
                .take_due(Instant::now())
                .map(|commit| commit.offsets),
            Some(HashMap::from([(p0, Offset(13))])),
            "the late completion releases the whole prefix"
        );
    }

    #[test]
    fn a_partition_with_a_held_prefix_does_not_hold_back_another() {
        let (ledger, settlement) = settlement();
        let p0 = TopicPartition::new("events", 0);
        let p1 = TopicPartition::new("events", 1);
        charge_offsets(&ledger, &p0, 0, &[10, 11]);
        charge_offsets(&ledger, &p1, 0, &[20, 21]);

        settlement
            .settle_run(&p0, 0, &[Offset(10), Offset(11)])
            .unwrap();
        settlement.settle_run(&p1, 0, &[Offset(21)]).unwrap();

        assert_eq!(
            settlement
                .take_due(Instant::now())
                .map(|commit| commit.offsets),
            Some(HashMap::from([(p0, Offset(12))])),
            "partition 1 waits for offset 20"
        );
    }

    #[test]
    fn a_run_settled_twice_fails_and_keeps_the_frontier_taken_before_it() {
        let (ledger, settlement) = settlement();
        let p0 = TopicPartition::new("events", 0);
        charge_offsets(&ledger, &p0, 0, &[10, 11]);
        settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();

        assert!(settlement.settle_run(&p0, 0, &[Offset(10)]).is_err());

        assert_eq!(
            settlement.take_all().map(|commit| commit.offsets),
            Some(HashMap::from([(p0, Offset(11))])),
            "offset 10 completed before the violation"
        );
    }

    #[test]
    fn a_stale_stamp_run_commits_nothing() {
        let (ledger, settlement) = settlement();
        let p0 = TopicPartition::new("events", 0);
        charge_offsets(&ledger, &p0, 0, &[10]);

        ledger.forget_partitions([("events", 0)]);
        charge_offsets(&ledger, &p0, 1, &[10]);

        settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();

        assert!(settlement.take_due(Instant::now()).is_none());
        assert_eq!(
            ledger.held(&p0).offsets,
            1,
            "the redelivered offset stays uncompleted"
        );
    }

    #[test]
    fn a_frontier_within_the_interval_waits_for_it_unless_everything_is_taken() {
        let ledger = Arc::new(TopicOffsetLedger::new());
        let settlement = GroupSettlement::new(
            Arc::clone(&ledger),
            Duration::from_secs(3600),
            BudgetLimits::unset(),
        );
        let p0 = TopicPartition::new("events", 0);
        charge_offsets(&ledger, &p0, 0, &[10, 11]);
        let now = Instant::now();

        settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();
        assert_eq!(
            settlement.take_due(now).map(|commit| commit.offsets),
            Some(HashMap::from([(p0.clone(), Offset(11))]))
        );

        settlement.settle_run(&p0, 0, &[Offset(11)]).unwrap();
        assert!(settlement.take_due(now).is_none());
        assert_eq!(
            settlement.take_all().map(|commit| commit.offsets),
            Some(HashMap::from([(p0, Offset(12))]))
        );
    }

    #[test]
    fn a_revoked_partitions_pending_frontier_is_not_committed() {
        let ledger = Arc::new(TopicOffsetLedger::new());
        let settlement = GroupSettlement::new(
            Arc::clone(&ledger),
            Duration::from_secs(3600),
            BudgetLimits::unset(),
        );
        let p0 = TopicPartition::new("events", 0);
        charge_offsets(&ledger, &p0, 0, &[10]);
        settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();
        assert!(settlement.take_due(Instant::now()).is_some());

        charge_offsets(&ledger, &p0, 0, &[11]);
        settlement.settle_run(&p0, 0, &[Offset(11)]).unwrap();
        settlement.on_revoked(&[p0]);

        assert!(settlement.take_all().is_none());
    }

    const HOUR: Duration = Duration::from_secs(3600);

    fn p0() -> TopicPartition {
        TopicPartition::new("events", 0)
    }

    fn budgeted_hourly_settlement_of_10_events_resuming_under_8() -> GroupSettlement {
        GroupSettlement::new(
            Arc::new(TopicOffsetLedger::new()),
            HOUR,
            BudgetLimits::new(NonZeroU64::new(10), None, 0.8).expect("valid limits"),
        )
    }

    fn charge(
        settlement: &GroupSettlement,
        topic_partition: &TopicPartition,
        offset: i64,
        events: u64,
    ) -> Option<Pause> {
        settlement
            .charge(
                topic_partition,
                0,
                &[(Offset(offset), Charge { events, bytes: 0 })],
            )
            .expect("the ledger accepts the charge")
            .pause
    }

    #[test]
    fn a_take_the_interval_holds_counts_toward_the_pause() {
        let settlement = budgeted_hourly_settlement_of_10_events_resuming_under_8();
        charge(&settlement, &p0(), 10, 6);
        settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();

        assert_eq!(charge(&settlement, &p0(), 11, 4), Some(Pause));
    }

    enum HandOut {
        DueAfterTheInterval,
        All,
    }

    #[rstest::rstest]
    #[case::due_after_the_interval(HandOut::DueAfterTheInterval)]
    #[case::all(HandOut::All)]
    fn a_paused_partition_resumes_only_when_its_frontier_is_handed_out(#[case] hand_out: HandOut) {
        let settlement = budgeted_hourly_settlement_of_10_events_resuming_under_8();
        let now = Instant::now();
        charge(&settlement, &p0(), 9, 1);
        settlement.settle_run(&p0(), 0, &[Offset(9)]).unwrap();
        settlement
            .take_due(now)
            .expect("the first frontier is due at once");
        assert_eq!(charge(&settlement, &p0(), 10, 10), Some(Pause));

        settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();
        assert!(settlement.take_due(now).is_none());

        let commit = match hand_out {
            HandOut::DueAfterTheInterval => settlement.take_due(now + HOUR),
            HandOut::All => settlement.take_all(),
        }
        .expect("the frontier is handed out");
        assert_eq!(commit.resume, vec![p0()]);
    }

    #[test]
    fn a_held_prefix_keeps_its_partition_paused_and_another_is_not_resumed() {
        let settlement = budgeted_hourly_settlement_of_10_events_resuming_under_8();
        let p1 = TopicPartition::new("events", 1);
        charge(&settlement, &p0(), 10, 5);
        assert_eq!(charge(&settlement, &p0(), 11, 5), Some(Pause));
        charge(&settlement, &p1, 20, 1);

        settlement.settle_run(&p0(), 0, &[Offset(11)]).unwrap();
        settlement.settle_run(&p1, 0, &[Offset(20)]).unwrap();
        let commit = settlement.take_all().expect("partition 1 is due");
        assert_eq!(commit.offsets, HashMap::from([(p1, Offset(21))]));
        assert!(
            commit.resume.is_empty(),
            "offset 10 still holds partition 0"
        );

        settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();
        assert_eq!(
            settlement.take_all().expect("partition 0 is due").resume,
            vec![p0()]
        );
    }

    #[test]
    fn a_revoked_paused_partition_is_never_resumed() {
        let settlement = budgeted_hourly_settlement_of_10_events_resuming_under_8();
        assert_eq!(charge(&settlement, &p0(), 10, 10), Some(Pause));
        settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();

        settlement.on_revoked(&[p0()]);

        assert!(settlement.take_all().is_none());
        settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();
        assert!(
            settlement.take_all().is_none(),
            "the stale run moves nothing"
        );
    }

    #[test]
    fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {
        let ledger = Arc::new(TopicOffsetLedger::new());
        let settlement = GroupSettlement::new(
            Arc::clone(&ledger),
            Duration::from_secs(3600),
            BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),
        );
        let p0 = TopicPartition::new("events", 0);
        let one_event = [(
            Offset(10),
            Charge {
                events: 1,
                bytes: 0,
            },
        )];
        assert_eq!(
            settlement
                .charge(&p0, 0, &one_event)
                .map(|charged| charged.pause),
            Ok(Some(Pause))
        );
        settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();

        settlement.on_assigned(std::slice::from_ref(&p0));

        assert!(settlement.take_all().is_none());
        assert_eq!(ledger.generation(&p0), 1);
        assert_eq!(
            settlement
                .charge(&p0, 1, &one_event)
                .map(|charged| charged.pause),
            Ok(Some(Pause)),
            "the regained partition starts unpaused"
        );
    }
}