Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.5 KB
Raw Blame
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;

use common_kafka_consumer::{
    GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,
};
use tokio::time::Instant;

use super::collector::tests::message;
use super::{CollectedBatch, InFlightPoll, Parts, PollLimits, PollingStrategy, Step};
use crate::partition_budget::BudgetLimits;
use crate::settlement::{GroupSettlement, Settlement};

pub(super) const BATCH_TIMEOUT: Duration = Duration::from_millis(500);

pub(super) struct Harness<P> {
    pub(super) poller: P,
    pub(super) settlement: Arc<GroupSettlement>,
    pub(super) ledger: Arc<TopicOffsetLedger>,
    pub(super) now: Instant,
}

impl<P: PollingStrategy> Harness<P> {
    pub(super) fn new(make: impl FnOnce(Parts) -> P) -> Self {
        let ledger = Arc::new(TopicOffsetLedger::new());
        let settlement = Arc::new(GroupSettlement::new(
            Arc::clone(&ledger),
            Duration::ZERO,
            BudgetLimits::unset(),
        ));
        let poller = make(Parts {
            limits: PollLimits {
                batch_size: 100,
                batch_size_bytes: 0,
                batch_timeout: BATCH_TIMEOUT,
            },
            settlement: Arc::clone(&settlement),
            group_id: "test".to_string(),
        });
        Self {
            poller,
            settlement,
            ledger,
            now: Instant::now(),
        }
    }

    pub(super) fn receive(&mut self, partition: i32, offsets: &[i64]) {
        for &offset in offsets {
            self.poller
                .on_received(&message(partition, offset, 10), self.now);
        }
    }

    pub(super) fn after_the_batch_timeout(&mut self) {
        self.now += BATCH_TIMEOUT;
    }

    pub(super) fn step(&mut self) -> Option<Step> {
        self.poller.next_step(self.now)
    }

    pub(super) fn submit(&mut self) -> HashMap<TopicPartition, (i64, i64)> {
        let Some(Step::Submit(collected)) = self.step() else {
            panic!("expected a poll to submit");
        };
        let CollectedBatch { mut partitions, .. } = collected;
        let mut uncommitted = HashMap::new();
        for (topic_partition, deliveries) in &mut partitions {
            deliveries.generation = self.ledger.generation(topic_partition);
            let charged = self
                .settlement
                .charge(topic_partition, deliveries.generation, &deliveries.charges)
                .expect("the ledger accepts the charge");
            uncommitted.insert(topic_partition.clone(), charged.uncommitted);
        }
        let spans = partitions
            .iter()
            .map(|(topic_partition, deliveries)| {
                (
                    topic_partition.clone(),
                    (deliveries.span.first, deliveries.span.last),
                )
            })
            .collect();
        self.poller.on_submitted(InFlightPoll {
            partitions,
            uncommitted,
        });
        spans
    }

    pub(super) fn complete(&mut self, partition: i32, offsets: &[i64]) {
        let generation = self.ledger.generation(&p(partition));
        self.complete_under(generation, partition, offsets);
    }

    pub(super) fn complete_under(&mut self, generation: u64, partition: i32, offsets: &[i64]) {
        self.poller
            .on_completed(&completion(generation, partition, offsets))
            .expect("the completion applies");
    }

    pub(super) fn revoke(&mut self, partition: i32) {
        self.settlement.on_revoked(&[p(partition)]);
        self.poller.on_revoked(&[p(partition)]);
    }

    pub(super) fn finish_drain(&mut self, partition: i32) -> HashMap<TopicPartition, Offset> {
        let committed = self
            .poller
            .on_drain_finished(&[p(partition)])
            .map(|commit| commit.offsets)
            .unwrap_or_default();
        self.settlement.on_revoked(&[p(partition)]);
        committed
    }

    pub(super) fn commit(&mut self) -> HashMap<TopicPartition, Offset> {
        match self.step() {
            Some(Step::Commit(commit)) => commit.offsets,
            _ => panic!("expected a commit"),
        }
    }
}

pub(super) fn completion(generation: u64, partition: i32, offsets: &[i64]) -> GroupCompletion {
    GroupCompletion {
        partition: Partition(partition),
        assignment_epoch: 1,
        generation,
        offsets: offsets.iter().map(|&offset| Offset(offset)).collect(),
        accepted: offsets.len() as u32,
    }
}

pub(super) fn p(partition: i32) -> TopicPartition {
    TopicPartition::new("events", partition)
}