:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Offset Tracker - Tracks offsets per partition for processing and checkpointing
//!
//! This component provides thread-safe offset tracking for:
//! - **Processed offsets**: Updated after batch processing succeeds, used for periodic commits
//! - **Committed offsets**: Updated after consumer.commit() succeeds, used for checkpointing
//! - **Producer offsets**: Tracks the highest offset written to the output topic, used for checkpointing
//!
//! The distinction between processed and committed is critical for disaster recovery:
//! if we crash between processing and committing, Kafka replays from the last committed offset.
//!
//! It also provides sequential batch IDs per partition to ensure ordering.
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use dashmap::DashMap;
use thiserror::Error;
use tracing::{debug, warn};
use crate::kafka::metrics_consts::OFFSET_TRACKER_OUT_OF_ORDER_BATCH;
use crate::kafka::types::Partition;
use crate::rebalance_tracker::RebalanceTracker;
/// Errors that can occur when retrieving committable offsets
#[derive(Error, Debug)]
pub enum OffsetTrackerError {
/// A rebalance is currently in progress - commits should be skipped
#[error("Rebalance in progress - offset commits should be skipped")]
RebalanceInProgress,
}
/// State tracked per partition
struct PartitionState {
/// The next offset to consume (highest processed + 1)
processed_offset: i64,
/// The last batch ID that was processed (for ordering verification)
last_processed_batch_id: u64,
/// The last successfully committed consumer offset (updated after consumer.commit())
/// Used for checkpointing - represents the offset Kafka would replay from on restart
committed_offset: i64,
/// The highest producer offset written to the output topic for this partition
/// Used for checkpointing to track output progress
producer_offset: i64,
}
/// Thread-safe tracker for processed offsets per partition
///
/// This is used to track the highest offset that has been successfully processed
/// for each partition. The consumer periodically queries this tracker to get
/// offsets that are safe to commit.
///
/// It also assigns sequential batch IDs to ensure ordering within partitions.
pub struct OffsetTracker {
/// Map of partition to state (offset + last batch ID)
partition_state: DashMap<Partition, PartitionState>,
/// Global counter for assigning batch IDs (unique across all partitions)
next_batch_id: AtomicU64,
/// Reference to rebalance coordinator for checking rebalance state.
/// This is a lightweight dependency (~8 bytes) vs the full StoreManager.
rebalance_tracker: Arc<RebalanceTracker>,
}
impl OffsetTracker {
/// Create a new offset tracker with the given rebalance coordinator.
///
/// The coordinator is used to check if a rebalance is in progress,
/// which blocks offset commits to prevent committing stale offsets.
pub fn new(rebalance_tracker: Arc<RebalanceTracker>) -> Self {
Self {
partition_state: DashMap::new(),
next_batch_id: AtomicU64::new(1), // Start at 1 so 0 can mean "no batch"
rebalance_tracker,
}
}
/// Check if a rebalance is currently in progress.
///
/// Delegates to the rebalance coordinator's counter, which correctly
/// handles overlapping rebalances using increment/decrement semantics.
pub fn is_rebalancing(&self) -> bool {
self.rebalance_tracker.is_rebalancing()
}
/// Assign a new batch ID for a partition.
///
/// Batch IDs are globally unique and monotonically increasing. They're used
/// to verify ordering when marking batches as processed.
pub fn assign_batch_id(&self) -> u64 {
self.next_batch_id.fetch_add(1, Ordering::SeqCst)
}
/// Mark a batch as processed.
///
/// The offset should be `last_offset + 1` (the next offset to consume).
/// The batch_id must be greater than the last processed batch_id for this
/// partition to ensure ordering. If a batch is processed out of order,
/// a warning is logged and a metric is emitted, but the offset is still
/// updated if it advances.
pub fn mark_processed(&self, partition: &Partition, batch_id: u64, next_offset: i64) {
self.partition_state
.entry(partition.clone())
.and_modify(|state| {
// Verify ordering
if batch_id <= state.last_processed_batch_id {
warn!(
topic = partition.topic(),
partition = partition.partition_number(),
batch_id = batch_id,
last_batch_id = state.last_processed_batch_id,
"Batch processed out of order"
);
// Emit metric for out-of-order batch processing
metrics::counter!(
OFFSET_TRACKER_OUT_OF_ORDER_BATCH,
"topic" => partition.topic().to_string(),
"partition" => partition.partition_number().to_string()
)
.increment(1);
}
// Only advance offset, never go backwards
if next_offset > state.processed_offset {
debug!(
topic = partition.topic(),
partition = partition.partition_number(),
batch_id = batch_id,
previous_offset = state.processed_offset,
new_offset = next_offset,
"Advancing processed offset"
);
state.processed_offset = next_offset;
}
// Always update last batch ID to track ordering
if batch_id > state.last_processed_batch_id {
state.last_processed_batch_id = batch_id;
}
})
.or_insert_with(|| {
debug!(
topic = partition.topic(),
partition = partition.partition_number(),
batch_id = batch_id,
offset = next_offset,
"Initializing partition state"
);
PartitionState {
processed_offset: next_offset,
last_processed_batch_id: batch_id,
committed_offset: 0,
producer_offset: 0,
}
});
}
/// Get all offsets ready for commit (thread-safe snapshot)
///
/// Returns a map of partition to next offset to consume. These offsets
/// can be safely committed to Kafka as they represent successfully
/// processed batches.
///
/// # Errors
///
/// Returns `OffsetTrackerError::RebalanceInProgress` if a rebalance is currently
/// in progress. Callers should skip committing offsets in this case.
pub fn get_committable_offsets(&self) -> Result<HashMap<Partition, i64>, OffsetTrackerError> {
// Check rebalancing state first - if true, return error to prevent commits
if self.is_rebalancing() {
return Err(OffsetTrackerError::RebalanceInProgress);
}
Ok(self
.partition_state
.iter()
.map(|entry| (entry.key().clone(), entry.value().processed_offset))
.collect())
}
/// Get the current offset for a specific partition
pub fn get_partition_offset(&self, partition: &Partition) -> Option<i64> {
self.partition_state
.get(partition)
.map(|r| r.value().processed_offset)
}
/// Mark offsets as successfully committed to Kafka
///
/// Called after `consumer.commit()` succeeds. These offsets are used for
/// checkpointing as they represent the true recovery point - if we crash,
/// Kafka will replay from the last committed offset.
///
/// # Arguments
/// * `offsets` - Map of partition to committed offset (next offset to consume)
pub fn mark_committed(&self, offsets: &HashMap<Partition, i64>) {
for (partition, offset) in offsets {
self.partition_state
.entry(partition.clone())
.and_modify(|state| {
if *offset > state.committed_offset {
debug!(
topic = partition.topic(),
partition = partition.partition_number(),
previous_committed = state.committed_offset,
new_committed = offset,
"Advancing committed offset"
);
state.committed_offset = *offset;
}
});
}
}
/// Get the committed offset for a specific partition
///
/// Returns the last successfully committed consumer offset, which represents
/// where Kafka would resume consumption after a restart.
pub fn get_committed_offset(&self, partition: &Partition) -> Option<i64> {
self.partition_state
.get(partition)
.map(|r| r.value().committed_offset)
}
/// Mark a producer offset for a partition
///
/// Called after successfully producing messages to the output topic.
/// Tracks the highest offset written so checkpointing knows output progress.
///
/// # Arguments
/// * `partition` - The input partition this producer write corresponds to
/// * `offset` - The producer offset returned from rdkafka
pub fn mark_produced(&self, partition: &Partition, offset: i64) {
self.partition_state
.entry(partition.clone())
.and_modify(|state| {
if offset > state.producer_offset {
debug!(
topic = partition.topic(),
partition = partition.partition_number(),
previous_producer = state.producer_offset,
new_producer = offset,
"Advancing producer offset"
);
state.producer_offset = offset;
}
});
}
/// Get the producer offset for a specific partition
///
/// Returns the highest offset written to the output topic for messages
/// originating from this input partition.
pub fn get_producer_offset(&self, partition: &Partition) -> Option<i64> {
self.partition_state
.get(partition)
.map(|r| r.value().producer_offset)
}
/// Clear offset tracking for a partition (during revocation)
///
/// Called when a partition is revoked to clean up state. The offset
/// should have been committed before calling this.
pub fn clear_partition(&self, partition: &Partition) {
if self.partition_state.remove(partition).is_some() {
debug!(
topic = partition.topic(),
partition = partition.partition_number(),
"Cleared offset tracking for revoked partition"
);
}
}
/// Initialize partition state from restored metadata (local or S3 import).
///
/// Populates committed and producer offsets so checkpointing reflects the pre-restart
/// state. Only inserts if no entry exists yet — safe to call multiple times.
pub fn init_partition_from_metadata(
&self,
partition: &Partition,
consumer_offset: i64,
producer_offset: i64,
) {
self.partition_state
.entry(partition.clone())
.or_insert_with(|| PartitionState {
processed_offset: consumer_offset,
last_processed_batch_id: 0,
committed_offset: consumer_offset,
producer_offset,
});
}
/// Clear all partitions (during shutdown)
pub fn clear_all(&self) {
self.partition_state.clear();
}
/// Get the number of partitions being tracked
pub fn partition_count(&self) -> usize {
self.partition_state.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::create_test_tracker;
fn test_partition(num: i32) -> Partition {
Partition::new("test-topic".to_string(), num)
}
#[test]
fn test_assign_batch_id_is_sequential() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let id1 = tracker.assign_batch_id();
let id2 = tracker.assign_batch_id();
let id3 = tracker.assign_batch_id();
assert_eq!(id1, 1);
assert_eq!(id2, 2);
assert_eq!(id3, 3);
}
#[test]
fn test_mark_processed_initializes_offset() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id, 100);
assert_eq!(tracker.get_partition_offset(&partition), Some(100));
}
#[test]
fn test_mark_processed_advances_offset() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id1 = tracker.assign_batch_id();
let batch_id2 = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id1, 100);
tracker.mark_processed(&partition, batch_id2, 150);
assert_eq!(tracker.get_partition_offset(&partition), Some(150));
}
#[test]
fn test_mark_processed_never_goes_backwards() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id1 = tracker.assign_batch_id();
let batch_id2 = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id1, 100);
tracker.mark_processed(&partition, batch_id2, 50); // Try to go backwards
// Should still be at 100
assert_eq!(tracker.get_partition_offset(&partition), Some(100));
}
#[test]
fn test_multiple_partitions() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let p0 = test_partition(0);
let p1 = test_partition(1);
let p2 = test_partition(2);
let batch_id1 = tracker.assign_batch_id();
let batch_id2 = tracker.assign_batch_id();
let batch_id3 = tracker.assign_batch_id();
tracker.mark_processed(&p0, batch_id1, 100);
tracker.mark_processed(&p1, batch_id2, 200);
tracker.mark_processed(&p2, batch_id3, 300);
let offsets = tracker.get_committable_offsets().unwrap();
assert_eq!(offsets.len(), 3);
assert_eq!(offsets.get(&p0), Some(&100));
assert_eq!(offsets.get(&p1), Some(&200));
assert_eq!(offsets.get(&p2), Some(&300));
}
#[test]
fn test_clear_partition() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let p0 = test_partition(0);
let p1 = test_partition(1);
let batch_id1 = tracker.assign_batch_id();
let batch_id2 = tracker.assign_batch_id();
tracker.mark_processed(&p0, batch_id1, 100);
tracker.mark_processed(&p1, batch_id2, 200);
tracker.clear_partition(&p0);
assert_eq!(tracker.get_partition_offset(&p0), None);
assert_eq!(tracker.get_partition_offset(&p1), Some(200));
assert_eq!(tracker.partition_count(), 1);
}
#[test]
fn test_clear_all() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let batch_id1 = tracker.assign_batch_id();
let batch_id2 = tracker.assign_batch_id();
let batch_id3 = tracker.assign_batch_id();
tracker.mark_processed(&test_partition(0), batch_id1, 100);
tracker.mark_processed(&test_partition(1), batch_id2, 200);
tracker.mark_processed(&test_partition(2), batch_id3, 300);
assert_eq!(tracker.partition_count(), 3);
tracker.clear_all();
assert_eq!(tracker.partition_count(), 0);
}
#[test]
fn test_concurrent_updates_same_partition() {
use std::thread;
let coordinator = create_test_tracker();
let tracker = Arc::new(OffsetTracker::new(coordinator));
let partition = test_partition(0);
// Pre-assign batch IDs so they're sequential
let batch_ids: Vec<u64> = (0..10).map(|_| tracker.assign_batch_id()).collect();
let mut handles = vec![];
// Spawn multiple threads that update the same partition
for (i, batch_id) in batch_ids.into_iter().enumerate() {
let tracker_clone = tracker.clone();
let partition_clone = partition.clone();
let offset = ((i + 1) * 100) as i64; // 100, 200, 300, ...
handles.push(thread::spawn(move || {
tracker_clone.mark_processed(&partition_clone, batch_id, offset);
}));
}
for handle in handles {
handle.join().unwrap();
}
// The highest offset should win (1000)
assert_eq!(tracker.get_partition_offset(&partition), Some(1000));
}
#[test]
fn test_concurrent_different_partitions() {
use std::thread;
let coordinator = create_test_tracker();
let tracker = Arc::new(OffsetTracker::new(coordinator));
// Pre-assign batch IDs
let batch_ids: Vec<u64> = (0..10).map(|_| tracker.assign_batch_id()).collect();
let mut handles = vec![];
// Spawn multiple threads, each updating a different partition
for (i, batch_id) in batch_ids.into_iter().enumerate() {
let tracker_clone = tracker.clone();
let partition = test_partition(i as i32);
let offset = (i as i64 + 1) * 100;
handles.push(thread::spawn(move || {
tracker_clone.mark_processed(&partition, batch_id, offset);
}));
}
for handle in handles {
handle.join().unwrap();
}
let offsets = tracker.get_committable_offsets().unwrap();
assert_eq!(offsets.len(), 10);
for i in 0..10 {
let partition = test_partition(i);
let expected = (i as i64 + 1) * 100;
assert_eq!(offsets.get(&partition), Some(&expected));
}
}
#[test]
fn test_out_of_order_batch_processing() {
// This tests the scenario where batches complete out of order
// (e.g., batch 2 finishes before batch 1)
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id1 = tracker.assign_batch_id(); // 1
let batch_id2 = tracker.assign_batch_id(); // 2
// Batch 2 completes first with higher offset
tracker.mark_processed(&partition, batch_id2, 200);
assert_eq!(tracker.get_partition_offset(&partition), Some(200));
// Batch 1 completes later with lower offset - should not regress
tracker.mark_processed(&partition, batch_id1, 100);
assert_eq!(tracker.get_partition_offset(&partition), Some(200));
}
#[test]
fn test_rebalancing_blocks_commits() {
// Test that rebalancing state (from coordinator) blocks commits
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator.clone());
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
// Initially not rebalancing
assert!(!tracker.is_rebalancing());
// Mark some offsets
tracker.mark_processed(&partition, batch_id, 100);
// Should be able to get offsets
let offsets = tracker.get_committable_offsets();
assert!(offsets.is_ok());
assert_eq!(offsets.unwrap().get(&partition), Some(&100));
// Start rebalancing via coordinator
coordinator.start_rebalancing();
assert!(tracker.is_rebalancing());
// Should get RebalanceInProgress error
let result = tracker.get_committable_offsets();
assert!(matches!(
result,
Err(OffsetTrackerError::RebalanceInProgress)
));
// End rebalancing via coordinator
coordinator.finish_rebalancing();
assert!(!tracker.is_rebalancing());
// Should be able to get offsets again
let offsets = tracker.get_committable_offsets();
assert!(offsets.is_ok());
assert_eq!(offsets.unwrap().get(&partition), Some(&100));
}
#[test]
fn test_mark_committed_updates_committed_offset() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
// First, mark as processed to create the partition state
tracker.mark_processed(&partition, batch_id, 100);
// Committed offset starts at 0
assert_eq!(tracker.get_committed_offset(&partition), Some(0));
// Mark as committed
let mut offsets = HashMap::new();
offsets.insert(partition.clone(), 100);
tracker.mark_committed(&offsets);
// Committed offset should now be 100
assert_eq!(tracker.get_committed_offset(&partition), Some(100));
}
#[test]
fn test_committed_offset_never_goes_backwards() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id, 200);
let mut offsets = HashMap::new();
offsets.insert(partition.clone(), 150);
tracker.mark_committed(&offsets);
assert_eq!(tracker.get_committed_offset(&partition), Some(150));
// Try to go backwards - should be ignored
offsets.insert(partition.clone(), 100);
tracker.mark_committed(&offsets);
assert_eq!(tracker.get_committed_offset(&partition), Some(150));
// Advance forward - should work
offsets.insert(partition.clone(), 200);
tracker.mark_committed(&offsets);
assert_eq!(tracker.get_committed_offset(&partition), Some(200));
}
#[test]
fn test_mark_produced_updates_producer_offset() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
// First, mark as processed to create the partition state
tracker.mark_processed(&partition, batch_id, 100);
// Producer offset starts at 0
assert_eq!(tracker.get_producer_offset(&partition), Some(0));
// Mark producer offset
tracker.mark_produced(&partition, 50);
assert_eq!(tracker.get_producer_offset(&partition), Some(50));
// Advance producer offset
tracker.mark_produced(&partition, 75);
assert_eq!(tracker.get_producer_offset(&partition), Some(75));
}
#[test]
fn test_producer_offset_never_goes_backwards() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id, 100);
tracker.mark_produced(&partition, 50);
assert_eq!(tracker.get_producer_offset(&partition), Some(50));
// Try to go backwards - should be ignored
tracker.mark_produced(&partition, 25);
assert_eq!(tracker.get_producer_offset(&partition), Some(50));
}
#[test]
fn test_committed_and_producer_offsets_independent() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id, 100);
// Set committed and producer offsets to different values
let mut offsets = HashMap::new();
offsets.insert(partition.clone(), 80);
tracker.mark_committed(&offsets);
tracker.mark_produced(&partition, 50);
// Verify they're tracked independently
assert_eq!(tracker.get_partition_offset(&partition), Some(100));
assert_eq!(tracker.get_committed_offset(&partition), Some(80));
assert_eq!(tracker.get_producer_offset(&partition), Some(50));
}
#[test]
fn test_clear_partition_clears_all_offsets() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
tracker.mark_processed(&partition, batch_id, 100);
let mut offsets = HashMap::new();
offsets.insert(partition.clone(), 80);
tracker.mark_committed(&offsets);
tracker.mark_produced(&partition, 50);
// Clear the partition
tracker.clear_partition(&partition);
// All offsets should be None
assert_eq!(tracker.get_partition_offset(&partition), None);
assert_eq!(tracker.get_committed_offset(&partition), None);
assert_eq!(tracker.get_producer_offset(&partition), None);
}
#[test]
fn test_init_partition_from_metadata_seeds_all_offsets() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
tracker.init_partition_from_metadata(&partition, 1000, 500);
assert_eq!(tracker.get_partition_offset(&partition), Some(1000));
assert_eq!(tracker.get_committed_offset(&partition), Some(1000));
assert_eq!(tracker.get_producer_offset(&partition), Some(500));
}
#[test]
fn test_init_partition_from_metadata_does_not_overwrite_existing_entry() {
let coordinator = create_test_tracker();
let tracker = OffsetTracker::new(coordinator);
let partition = test_partition(0);
let batch_id = tracker.assign_batch_id();
// Entry already exists from normal processing
tracker.mark_processed(&partition, batch_id, 2000);
// init should be a no-op since the entry already exists
tracker.init_partition_from_metadata(&partition, 1000, 500);
assert_eq!(tracker.get_partition_offset(&partition), Some(2000));
}
}