Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion-consumer): count a taken frontier as uncommitted until its commit is submitted (#118691)

Paweł Ledwońf3cfdb1c215d
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 5879b1dd25fd..6a270db41d83 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -6370,6 +6370,7 @@ dependencies = [
"lifecycle",
"metrics",
"metrics-exporter-prometheus",
+ "metrics-util",
"prost 0.13.5",
"rand 0.8.5",
"rdkafka",
diff --git a/rust/common/kafka-consumer/README.md b/rust/common/kafka-consumer/README.md
index 94ff54845a93..fcb010d4f67d 100644
--- a/rust/common/kafka-consumer/README.md
+++ b/rust/common/kafka-consumer/README.md
@@ -17,8 +17,5 @@ Kafka consumption primitives shared by stateful services.
| Metric | Labels | Meaning |
| --- | --- | --- |
-| `kafka_consumer_ledger_uncommitted_offsets` (gauge) | `topic`, `partition` | Offsets the partition's window holds. |
-| `kafka_consumer_ledger_uncommitted_events` (gauge) | `topic`, `partition` | Events those offsets carry. |
-| `kafka_consumer_ledger_uncommitted_bytes` (gauge) | `topic`, `partition` | Bytes those offsets carry (payload plus key plus headers). |
| `kafka_consumer_ledger_stale_slices_total` (counter) | `stage` | Charges and settlements dropped because their partition was reassigned while they were in flight; a few around a rebalance are expected. |
| `kafka_consumer_ledger_errors_total` (counter) | `stage`, `kind` | Contract violations in the accounting; must stay 0. A violation resets that partition's ledger and the consumer keeps running. |
diff --git a/rust/common/kafka-consumer/src/partition_offset_ledger.rs b/rust/common/kafka-consumer/src/partition_offset_ledger.rs
index e54ca6d1015a..f6b20e98b268 100644
--- a/rust/common/kafka-consumer/src/partition_offset_ledger.rs
+++ b/rust/common/kafka-consumer/src/partition_offset_ledger.rs
@@ -1,5 +1,6 @@
use std::collections::VecDeque;
use std::fmt;
+use std::ops::{Add, AddAssign};
use crate::charge::Charge;
use crate::types::Offset;
@@ -79,14 +80,29 @@ pub struct TakenFrontier {
pub gap_offset_count: usize,
}
-/// What a window holds: the offsets charged and not yet drained by
-/// `take_frontier`, and the charge they carry.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Held {
pub offsets: usize,
pub charge: Charge,
}
+impl Add for Held {
+ type Output = Held;
+
+ fn add(self, rhs: Held) -> Held {
+ Held {
+ offsets: self.offsets + rhs.offsets,
+ charge: self.charge + rhs.charge,
+ }
+ }
+}
+
+impl AddAssign for Held {
+ fn add_assign(&mut self, rhs: Held) {
+ *self = *self + rhs;
+ }
+}
+
/// Per-partition offset accounting: record offsets as they are delivered,
/// complete them in any order, and read the frontier in Kafka's
/// committed-offset representation. Observing the frontier never changes it;
diff --git a/rust/common/kafka-consumer/src/topic_offset_ledger.rs b/rust/common/kafka-consumer/src/topic_offset_ledger.rs
index 0e33aaeeb3eb..435da74605e0 100644
--- a/rust/common/kafka-consumer/src/topic_offset_ledger.rs
+++ b/rust/common/kafka-consumer/src/topic_offset_ledger.rs
@@ -2,19 +2,13 @@ use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
-use metrics::{counter, gauge};
+use metrics::counter;
use tracing::warn;
use crate::charge::Charge;
use crate::partition_offset_ledger::{Held, LedgerError, PartitionOffsetLedger, TakenFrontier};
use crate::types::Offset;
-/// Offsets a partition holds uncommitted.
-const UNCOMMITTED_OFFSETS: &str = "kafka_consumer_ledger_uncommitted_offsets";
-/// Events those offsets carry.
-const UNCOMMITTED_EVENTS: &str = "kafka_consumer_ledger_uncommitted_events";
-/// Bytes those offsets carry.
-const UNCOMMITTED_BYTES: &str = "kafka_consumer_ledger_uncommitted_bytes";
/// Charges and settlements dropped because their partition was reassigned
/// while they were in flight.
const STALE_SLICES: &str = "kafka_consumer_ledger_stale_slices_total";
@@ -66,9 +60,7 @@ pub enum Rejection {
/// generation is stale. Generations are independent per partition, so one
/// partition's reassignment never affects another's in-flight work.
///
-/// The ledger publishes its own metrics under the `kafka_consumer_ledger_`
-/// prefix, so every consumer built on this crate reports the same series and
-/// no caller can forget a gauge.
+/// The ledger counts its rejections and the gap offsets a take walks over.
#[derive(Default)]
pub struct TopicOffsetLedger {
partitions: Mutex<HashMap<TopicPartition, PartitionOffsetLedger>>,
@@ -105,7 +97,7 @@ impl TopicOffsetLedger {
/// Record one slice of delivered offsets on the partition's ledger,
/// founding the ledger when the slice is the partition's first delivery.
/// Returns what the partition's window holds after the charge, or why
- /// the slice was rejected. Publishes the outcome either way.
+ /// the slice was rejected.
pub fn charge(
&self,
topic_partition: &TopicPartition,
@@ -137,10 +129,8 @@ impl TopicOffsetLedger {
}
}
};
- // Outside the lock, and on every path, so no caller has to remember.
- match result {
- Ok(held) => publish_held(topic_partition, held),
- Err(rejection) => count_rejection("charge", rejection),
+ if let Err(rejection) = result {
+ count_rejection("charge", rejection);
}
result
}
@@ -187,8 +177,6 @@ impl TopicOffsetLedger {
}
}
};
- // The window only changes depth at a charge or a take, so a
- // settlement reports rejections and leaves the gauges alone.
if let Err(rejection) = result {
count_rejection("settle", rejection);
}
@@ -198,16 +186,14 @@ impl TopicOffsetLedger {
/// Drain the completed prefix and return the span it covers: the window
/// base it started from and the frontier it reached. `None` when the
/// partition has no ledger or nothing has completed at the front of its
- /// window. The drained offsets leave the window, so the gauges follow
- /// them down.
+ /// window.
pub fn take_frontier(&self, topic_partition: &TopicPartition) -> Option<TakenFrontier> {
- let (taken, held) = {
- let mut partitions = self.partitions.lock().unwrap();
- let ledger = partitions.get_mut(topic_partition)?;
- let taken = ledger.take_frontier()?;
- (taken, ledger.held())
- };
- publish_held(topic_partition, held);
+ let taken = self
+ .partitions
+ .lock()
+ .unwrap()
+ .get_mut(topic_partition)?
+ .take_frontier()?;
report_gap_offsets(topic_partition, &taken);
Some(taken)
}
@@ -226,56 +212,27 @@ impl TopicOffsetLedger {
/// when a partition is revoked or assigned: Kafka redelivers it from the
/// committed offset, which a kept ledger would reject as duplicate
/// delivery, and work still in flight from the old assignment must not
- /// land on the new one. The dropped offsets leave the window, so the
- /// partition's gauges go to zero.
+ /// land on the new one.
pub fn forget_partitions<'a>(
&self,
topic_partitions: impl IntoIterator<Item = (&'a str, i32)>,
) {
- let mut forgotten = Vec::new();
- {
- let mut partitions = self.partitions.lock().unwrap();
- for (topic, partition) in topic_partitions {
- let key = TopicPartition::new(topic, partition);
- let generation = partitions
- .get(&key)
- .map(PartitionOffsetLedger::generation)
- .unwrap_or_default();
- partitions.insert(key.clone(), PartitionOffsetLedger::new(generation + 1));
- forgotten.push(key);
- }
- self.generations_version
- .fetch_add(forgotten.len() as u64, Ordering::Relaxed);
- }
- for topic_partition in forgotten {
- publish_held(&topic_partition, Held::default());
+ let mut partitions = self.partitions.lock().unwrap();
+ let mut forgotten = 0;
+ for (topic, partition) in topic_partitions {
+ let key = TopicPartition::new(topic, partition);
+ let generation = partitions
+ .get(&key)
+ .map(PartitionOffsetLedger::generation)
+ .unwrap_or_default();
+ partitions.insert(key, PartitionOffsetLedger::new(generation + 1));
+ forgotten += 1;
}
+ self.generations_version
+ .fetch_add(forgotten, Ordering::Relaxed);
}
}
-/// Publish what a partition's window holds.
-fn publish_held(topic_partition: &TopicPartition, held: Held) {
- let (topic, partition) = labels(topic_partition);
- gauge!(
- UNCOMMITTED_OFFSETS,
- "topic" => topic.clone(),
- "partition" => partition.clone()
- )
- .set(held.offsets as f64);
- gauge!(
- UNCOMMITTED_EVENTS,
- "topic" => topic.clone(),
- "partition" => partition.clone()
- )
- .set(held.charge.events as f64);
- gauge!(
- UNCOMMITTED_BYTES,
- "topic" => topic,
- "partition" => partition
- )
- .set(held.charge.bytes as f64);
-}
-
fn report_gap_offsets(topic_partition: &TopicPartition, taken: &TakenFrontier) {
if taken.gap_offset_count == 0 {
return;
diff --git a/rust/ingestion-consumer/Cargo.toml b/rust/ingestion-consumer/Cargo.toml
index 20230b74e5d7..1f6fe1ec3ad9 100644
--- a/rust/ingestion-consumer/Cargo.toml
+++ b/rust/ingestion-consumer/Cargo.toml
@@ -39,6 +39,7 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
[dev-dependencies]
+metrics-util = { workspace = true }
rstest = "0.26"
tokio = { workspace = true, features = ["test-util"] }
uuid = { workspace = true }
diff --git a/rust/ingestion-consumer/README.md b/rust/ingestion-consumer/README.md
index 3de9c7c24179..7c91dafdd58e 100644
--- a/rust/ingestion-consumer/README.md
+++ b/rust/ingestion-consumer/README.md
@@ -44,8 +44,9 @@ At `group` granularity the pacer keeps the latest frontier per partition and han
The consumer asks the pacer on each completion wake, at the top of each loop turn, and on the completion wait's one-second heartbeat, and commits everything pending on exit.
Admission is still per poll at both granularities: `CONSUMER_MAX_BACKGROUND_TASKS` caps in-flight polls, and a slot frees when its whole poll is covered, oldest first, so a stalled key still holds its poll's slot.
At `poll` granularity, `ingestion_consumer_commits_skipped_total{reason}` counts polls that settle with no frontier: `rejected` means the ledger dropped every slice (expected around a rebalance) and `no_frontier` means an earlier batch is still incomplete at the front of every window the batch settled.
-The ledger emits its own metrics, so any consumer built on the crate reports the same series.
-`kafka_consumer_ledger_uncommitted_offsets{topic,partition}` gauges each partition's window depth; `kafka_consumer_ledger_uncommitted_events` and `kafka_consumer_ledger_uncommitted_bytes` gauge the charge those offsets carry, where bytes is the payload plus key plus headers of each message.
+The settlement publishes the uncommitted gauges, because it holds both places a partition's charge can be: the ledger's window and the pacer.
+`kafka_consumer_ledger_uncommitted_offsets{topic,partition}` gauges the offsets each partition has polled and no commit has covered yet, including those taken for a commit the pacer still holds; `kafka_consumer_ledger_uncommitted_events` and `kafka_consumer_ledger_uncommitted_bytes` gauge the charge those offsets carry, where bytes is the payload plus key plus headers of each message.
+The ledger emits its own counters, so any consumer built on the crate reports the same series.
`kafka_consumer_ledger_stale_slices_total{stage}` counts charges and settlements dropped because their partition was reassigned while they were in flight; a few around a rebalance are expected.
`kafka_consumer_ledger_errors_total{stage,kind}` counts contract violations in the ledger's accounting; it must stay 0. A violation stops the consumer, and the restart replays from the last commit: the ledger resets the partition's window, so offsets in flight below its next delivery would never complete and a later commit would move past them. The consumer logs the rejected slice with the batch and ledger generations and the window depth before it stops.
diff --git a/rust/ingestion-consumer/docs/driver-model-plan.md b/rust/ingestion-consumer/docs/driver-model-plan.md
index d4b7f40faed0..a701273d8540 100644
--- a/rust/ingestion-consumer/docs/driver-model-plan.md
+++ b/rust/ingestion-consumer/docs/driver-model-plan.md
@@ -607,32 +607,30 @@ Exit criterion at the canary: zero ledger errors and gap walks, and a stalled pa
Outcome: no partition's uncommitted work exceeds its budget, in events and bytes. The in-flight admission cap is gone. Needs cycles 2 and 8.
-**Verify:** change 23's gauges run on all lanes with the budget unset.
-Exit criterion: `ingestion_consumer_budget_outstanding_events` summed per pod tracks the admission cap's occupancy, and `ingestion_consumer_budget_paused_partitions` stays flat at zero.
+**Verify:** the ledger's uncommitted gauges run on all lanes with no budget set.
+Exit criterion: `kafka_consumer_ledger_uncommitted_events` summed per pod tracks the admission cap's occupancy.
The budget is per partition, and so is the pause. One budget for the whole assignment would let a stalled partition fill it and pause every other partition with it, which undoes cycle 8. Per-partition commits are a hard prerequisite for the same reason: at `poll` granularity a stalled partition holds the commits, and so the refunds, of every partition in its poll, so every budget fills together.
-### 23. Add the budget accounting (implement)
+### 23. Make the uncommitted accounting exact (implement)
-**Task:** Track each partition's uncommitted work in events and bytes. Ship with no budget set: the admission cap still governs alone.
+**Task:** Count each partition's uncommitted work in events and bytes exactly: everything polled and not yet covered by a submitted commit. No budget exists yet: the admission cap still governs alone.
-**Goal:** The budget exists and reports before it constrains.
+**Goal:** The number the budget will bound exists and reports before anything constrains it.
- Charge each message to its partition on poll. The ledger slots already carry the costs (change 2).
-- Refund the charge when the commit that covers the message is issued. A revoked partition refunds everything it holds.
-- Gauge the outstanding charge per partition.
-- New config: the budget per partition in events and in bytes, and the low watermark as a fraction of the budget (default 0.8). Unset means no constraint.
-- The budget is a fixed amount per partition, not a share of a pod-wide amount. A rebalance changes how many partitions a pod holds, not how much each one may expose, so the value does not move at rebalances.
+- Refund the charge when the commit that covers the message goes out. The charge is always in one place: the ledger's window until a take, then the pacer with the frontier it covers until the settlement hands the frontier out, which is when the consumer submits the commit. Before this change the pacer kept only the offset, so the take itself was the refund and under the interval pacer the gauges ran up to one interval ahead of the commits. A revoked partition drops everything it holds in both places.
+- The settlement gauges the result per partition, since it alone sees both places. The budget's config and its pause gauges arrive with change 24, where they are read.
**Interfaces:**
-- Modify `Config`: add the per-partition budget, default unset.
-- Modify `IngestionConsumer`: track charge and refund through `OffsetLedger`. No polling change while the budget is unset.
+- Modify `CommitPacer`: keep each partition's pending frontier with the charge of every take since the last commit.
+- Modify `Settlement`: `charge` charges the ledger and returns what the partition holds uncommitted; `forget_partitions` forgets the ledger too. Both granularities move charge between the ledger and the pacer under one lock and publish the gauges.
+- Modify `TopicOffsetLedger`: stop publishing the uncommitted gauges.
**Metrics:**
-- Add `ingestion_consumer_budget_outstanding_events{topic,partition}` and `ingestion_consumer_budget_outstanding_bytes{topic,partition}` (gauges).
-- Add `ingestion_consumer_budget_paused_partitions` (gauge) and `ingestion_consumer_budget_pauses_total{topic,partition}` (counter). Both stay flat until change 24.
+- `kafka_consumer_ledger_uncommitted_offsets{topic,partition}`, `kafka_consumer_ledger_uncommitted_events{topic,partition}` and `kafka_consumer_ledger_uncommitted_bytes{topic,partition}` now include takes whose commit has not gone out, and the settlement publishes them. No new series.
### 24. Enable the budget (switchover)
@@ -646,9 +644,16 @@ The budget is per partition, and so is the pause. One budget for the whole assig
- Keep polling while partitions are paused. A paused partition still serves rebalance callbacks and keeps `max.poll.interval.ms` alive; an unpolled consumer does neither. An assignment resets librdkafka's pause flags, so a partition that arrives starts unpaused, with a fresh ledger generation and zero charge.
- Watch the paused-partitions gauge and `ingestion_lag_ms`. Rollback is unsetting the values.
+- The budget is a fixed amount per partition, not a share of a pod-wide amount. A rebalance changes how many partitions a pod holds, not how much each one may expose, so the value does not move at rebalances.
+
**Interfaces:**
-- Modify `IngestionConsumer`: pause and resume one partition at a time through the Kafka client.
+- Modify `Config`: add the budget per partition in events and in bytes, and the low watermark as a fraction of the budget (default 0.8). Unset means no constraint.
+- Modify `IngestionConsumer`: pause and resume one partition at a time through the Kafka client, from what the settlement reports uncommitted per partition.
+
+**Metrics:**
+
+- Add `ingestion_consumer_budget_paused_partitions` (gauge) and `ingestion_consumer_budget_pauses_total{topic,partition}` (counter).
### 25. Remove the in-flight admission cap (cleanup)
@@ -743,7 +748,7 @@ The steady-state dashboard after the migration:
- Delivery: `ingestion_lag_ms`, the `ingestion_consumer_messages_processed_total` rate, and the two sentinel violation counters, which must stay at zero.
- Commits: `kafka_consumer_ledger_uncommitted_offsets` per partition (frontier lag, and the ring-leak detector) and the `ingestion_consumer_offset_commits_total` rate.
- Scheduling: the key-table gauges and `ingestion_consumer_parked_retries_total`.
-- Admission: the budget gauges and `ingestion_consumer_budget_paused`.
+- Admission: `kafka_consumer_ledger_uncommitted_events` and `kafka_consumer_ledger_uncommitted_bytes` per partition, and `ingestion_consumer_budget_paused_partitions`.
- Requests: the request-size histograms, `ingestion_consumer_transport_duration_seconds`, `ingestion_consumer_governor_permits`, `ingestion_consumer_request_rtt_seconds`, and `ingestion_consumer_request_budget_partials_total`.
- Rebalance: `ingestion_consumer_drain_duration_seconds` and `ingestion_consumer_drain_dropped_messages_total`.
diff --git a/rust/ingestion-consumer/src/commit_pacer.rs b/rust/ingestion-consumer/src/commit_pacer.rs
index 44beefe3c6b8..4a46756a79cc 100644
--- a/rust/ingestion-consumer/src/commit_pacer.rs
+++ b/rust/ingestion-consumer/src/commit_pacer.rs
@@ -2,24 +2,24 @@
//! partition's frontier as it takes it from the ledger; the consumer asks the
//! pacer what is due and commits that in one call. The pacer holds no I/O.
//!
-//! The pacer keeps the latest frontier per partition and hands them out at
-//! most once per interval. A zero interval makes every take due.
+//! The pacer keeps the latest frontier per partition, with the charge of every
+//! take since the last commit, and hands them out at most once per interval.
+//! A zero interval makes every take due.
use std::collections::HashMap;
-use std::sync::Mutex;
use std::time::{Duration, Instant};
-use common_kafka_consumer::{Offset, TakenFrontier, TopicPartition};
+use common_kafka_consumer::{Held, Offset, TakenFrontier, TopicPartition};
pub struct CommitPacer {
interval: Duration,
- pending: Mutex<Pending>,
+ frontiers: HashMap<TopicPartition, PendingFrontier>,
+ last_take: Option<Instant>,
}
-#[derive(Default)]
-struct Pending {
- frontiers: HashMap<TopicPartition, Offset>,
- last_take: Option<Instant>,
+struct PendingFrontier {
+ offset: Offset,
+ held: Held,
}
impl CommitPacer {
@@ -30,51 +30,70 @@ impl CommitPacer {
pub fn every(interval: Duration) -> Self {
Self {
interval,
- pending: Mutex::new(Pending::default()),
+ frontiers: HashMap::new(),
+ last_take: None,
}
}
/// Replace the partition's pending offset; frontiers only move forward.
- pub fn advance_frontier(&self, topic_partition: &TopicPartition, taken: TakenFrontier) {
- self.pending
- .lock()
- .unwrap()
+ pub fn advance_frontier(&mut self, topic_partition: &TopicPartition, taken: TakenFrontier) {
+ let taken_offsets = usize::try_from(taken.offset - taken.first)
+ .expect("a frontier is above its window base");
+ let pending = self
.frontiers
- .insert(topic_partition.clone(), taken.offset);
+ .entry(topic_partition.clone())
+ .or_insert(PendingFrontier {
+ offset: taken.offset,
+ held: Held::default(),
+ });
+ pending.offset = taken.offset;
+ pending.held += Held {
+ offsets: taken_offsets,
+ charge: taken.charge,
+ };
+ }
+
+ pub fn pending(&self, topic_partition: &TopicPartition) -> Held {
+ self.frontiers
+ .get(topic_partition)
+ .map(|pending| pending.held)
+ .unwrap_or_default()
}
/// Partitions leaving the assignment: drop whatever is held for them. A
/// commit issued for a partition another member now owns could move the
/// group's offset back behind that member's progress, so the frontier
/// goes with the partition.
- pub fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
- let mut pending = self.pending.lock().unwrap();
+ pub fn forget_partitions(&mut self, topic_partitions: &[TopicPartition]) {
for topic_partition in topic_partitions {
- pending.frontiers.remove(topic_partition);
+ self.frontiers.remove(topic_partition);
}
}
- pub fn take_due(&self, now: Instant) -> Option<HashMap<TopicPartition, Offset>> {
- let mut pending = self.pending.lock().unwrap();
- if pending.frontiers.is_empty() {
+ pub fn take_due(&mut self, now: Instant) -> Option<HashMap<TopicPartition, Offset>> {
+ if self.frontiers.is_empty() {
return None;
}
- if pending
+ if self
.last_take
.is_some_and(|last| now.duration_since(last) < self.interval)
{
return None;
}
- pending.last_take = Some(now);
- Some(std::mem::take(&mut pending.frontiers))
+ self.last_take = Some(now);
+ self.take_all()
}
- pub fn take_all(&self) -> Option<HashMap<TopicPartition, Offset>> {
- let mut pending = self.pending.lock().unwrap();
- if pending.frontiers.is_empty() {
+ pub fn take_all(&mut self) -> Option<HashMap<TopicPartition, Offset>> {
+ if self.frontiers.is_empty() {
return None;
}
- Some(std::mem::take(&mut pending.frontiers))
+ Some(
+ self.frontiers
+ .drain()
+ .map(|(topic_partition, pending)| (topic_partition, pending.offset))
+ .collect(),
+ )
}
}
@@ -106,7 +125,7 @@ mod tests {
#[test]
fn a_pending_frontier_is_due_on_the_next_take() {
- let pacer = CommitPacer::immediate();
+ let mut pacer = CommitPacer::immediate();
pacer.advance_frontier(&tp(0), taken(0, 10));
let offsets = pacer.take_due(Instant::now()).expect("due");
@@ -119,7 +138,7 @@ mod tests {
#[test]
fn the_latest_frontier_per_partition_is_what_goes_out() {
- let pacer = CommitPacer::immediate();
+ let mut pacer = CommitPacer::immediate();
pacer.advance_frontier(&tp(0), taken(0, 10));
pacer.advance_frontier(&tp(1), taken(0, 20));
pacer.advance_frontier(&tp(0), taken(10, 12));
@@ -133,7 +152,7 @@ mod tests {
#[test]
fn a_forgotten_partition_is_not_committed() {
- let pacer = CommitPacer::immediate();
+ let mut pacer = CommitPacer::immediate();
pacer.advance_frontier(&tp(0), taken(0, 10));
pacer.advance_frontier(&tp(1), taken(0, 20));
@@ -147,7 +166,7 @@ mod tests {
#[test]
fn frontiers_handed_over_within_the_interval_wait_for_it() {
- let pacer = CommitPacer::every(Duration::from_millis(500));
+ let mut pacer = CommitPacer::every(Duration::from_millis(500));
let start = Instant::now();
pacer.advance_frontier(&tp(0), taken(0, 10));
@@ -173,7 +192,7 @@ mod tests {
#[test]
fn take_all_ignores_the_interval() {
- let pacer = CommitPacer::every(Duration::from_secs(3600));
+ let mut pacer = CommitPacer::every(Duration::from_secs(3600));
let start = Instant::now();
pacer.advance_frontier(&tp(0), taken(0, 10));
pacer.take_due(start).expect("due");
diff --git a/rust/ingestion-consumer/src/consumer.rs b/rust/ingestion-consumer/src/consumer.rs
index 0dc0a0525ffe..e7cb1f5ac850 100644
--- a/rust/ingestion-consumer/src/consumer.rs
+++ b/rust/ingestion-consumer/src/consumer.rs
@@ -231,19 +231,6 @@ struct RevokedPartition {
/// The consumer's revocation hook: purge the batcher, and record the revoked
/// partitions, because the purge means their polls never complete.
-fn forget_partitions(
- ledger: &TopicOffsetLedger,
- settlement: &dyn Settlement,
- partitions: &[TopicPartition],
-) {
- ledger.forget_partitions(
- partitions
- .iter()
- .map(|tp| (tp.topic.as_str(), tp.partition)),
- );
- settlement.forget_partitions(partitions);
-}
-
fn revoke_hook(
revoker: Revoker,
ledger: Arc<TopicOffsetLedger>,
@@ -251,7 +238,7 @@ fn revoke_hook(
revoked_partitions: Arc<Mutex<Vec<RevokedPartition>>>,
) -> PartitionHook {
Box::new(move |partitions| {
- forget_partitions(&ledger, &*settlement, partitions);
+ settlement.forget_partitions(partitions);
let pairs: Vec<(String, i32)> = partitions
.iter()
.map(|tp| (tp.topic.clone(), tp.partition))
@@ -268,13 +255,12 @@ fn revoke_hook(
}
fn assign_hook(
- ledger: Arc<TopicOffsetLedger>,
settlement: Arc<dyn Settlement>,
assignment_epoch: AssignmentEpoch,
) -> PartitionHook {
Box::new(move |partitions| {
// Also covers a loss with no revoke callback (an error rebalance, a fenced member).
- forget_partitions(&ledger, &*settlement, partitions);
+ settlement.forget_partitions(partitions);
assignment_epoch.bump();
})
}
@@ -293,11 +279,7 @@ fn wire_rebalance_hooks(
Arc::clone(settlement),
Arc::clone(&revoked_partitions),
));
- context.set_on_assign(assign_hook(
- Arc::clone(ledger),
- Arc::clone(settlement),
- assignment_epoch,
- ));
+ context.set_on_assign(assign_hook(Arc::clone(settlement), assignment_epoch));
revoked_partitions
}
@@ -1047,14 +1029,12 @@ impl IngestionConsumer {
// One ledger call per partition keeps the lock and the gauge labels
// off the per-message path.
for (topic_partition, partition) in &partitions {
- // The ledger counts both outcomes and publishes what the window
- // holds; a rejection is logged here, where the slice is still
- // known.
- if let Err(rejection) = self.topic_offset_ledger.charge(
- topic_partition,
- partition.generation,
- partition.charges.iter().copied(),
- ) {
+ // The ledger counts a rejection; it is logged here, where the
+ // slice is still known.
+ if let Err(rejection) =
+ self.settlement
+ .charge(topic_partition, partition.generation, &partition.charges)
+ {
report_rejection(
"charge",
topic_partition,
@@ -1271,7 +1251,7 @@ mod tests {
Duration::from_secs(3600),
));
let epoch = AssignmentEpoch::new();
- let on_assign = assign_hook(Arc::clone(&ledger), Arc::clone(&settlement), epoch.clone());
+ let on_assign = assign_hook(Arc::clone(&settlement), epoch.clone());
let p0 = TopicPartition::new("events", 0);
charge_offsets(&ledger, &p0, ledger.generation(&p0), &[10]);
settlement
diff --git a/rust/ingestion-consumer/src/settlement/group.rs b/rust/ingestion-consumer/src/settlement/group.rs
index 7152349136ed..3f925098cf56 100644
--- a/rust/ingestion-consumer/src/settlement/group.rs
+++ b/rust/ingestion-consumer/src/settlement/group.rs
@@ -2,45 +2,47 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
-use common_kafka_consumer::{Offset, TopicOffsetLedger, TopicPartition};
+use common_kafka_consumer::{Charge, Held, Offset, Rejection, TopicOffsetLedger, TopicPartition};
-use super::{ChargedSlice, Settlement};
+use super::{ChargedSlice, Settlement, Uncommitted};
use crate::commit_pacer::CommitPacer;
use crate::ledger_rejection::{report_rejection, RejectedSlice};
use crate::order_sentinel::OffsetSpan;
pub(crate) struct GroupSettlement {
- ledger: Arc<TopicOffsetLedger>,
- pacer: Arc<CommitPacer>,
+ uncommitted: Uncommitted,
}
impl GroupSettlement {
pub(crate) fn new(ledger: Arc<TopicOffsetLedger>, commit_interval: Duration) -> Self {
Self {
- ledger,
- pacer: Arc::new(CommitPacer::every(commit_interval)),
+ uncommitted: Uncommitted::new(ledger, CommitPacer::every(commit_interval)),
}
}
}
impl Settlement for GroupSettlement {
+ fn charge(
+ &self,
+ topic_partition: &TopicPartition,
+ generation: u64,
+ charges: &[(Offset, Charge)],
+ ) -> Result<Held, Rejection> {
+ self.uncommitted
+ .charge(topic_partition, generation, charges)
+ }
+
fn settle_run(
&self,
topic_partition: &TopicPartition,
generation: u64,
offsets: &[Offset],
) -> anyhow::Result<()> {
- match self
- .ledger
- .settle(topic_partition, generation, offsets.iter().copied())
+ if let Err(rejection) =
+ self.uncommitted
+ .settled(topic_partition, generation, offsets.iter().copied())
{
- Ok(Some(_)) => {
- if let Some(taken) = self.ledger.take_frontier(topic_partition) {
- self.pacer.advance_frontier(topic_partition, taken);
- }
- }
- Ok(None) => {}
- Err(rejection) => report_rejection(
+ report_rejection(
"settle",
topic_partition,
rejection,
@@ -48,7 +50,7 @@ impl Settlement for GroupSettlement {
first: offsets[0].0,
last: offsets[offsets.len() - 1].0,
}),
- )?,
+ )?;
}
Ok(())
}
@@ -58,15 +60,15 @@ impl Settlement for GroupSettlement {
}
fn take_due(&self, now: Instant) -> Option<HashMap<TopicPartition, Offset>> {
- self.pacer.take_due(now)
+ self.uncommitted.take_due(now)
}
fn take_all(&self) -> Option<HashMap<TopicPartition, Offset>> {
- self.pacer.take_all()
+ self.uncommitted.take_all()
}
fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
- self.pacer.forget_partitions(topic_partitions);
+ self.uncommitted.forget_partitions(topic_partitions);
}
}
diff --git a/rust/ingestion-consumer/src/settlement/mod.rs b/rust/ingestion-consumer/src/settlement/mod.rs
index d9c38a4f3da0..890cd3913748 100644
--- a/rust/ingestion-consumer/src/settlement/mod.rs
+++ b/rust/ingestion-consumer/src/settlement/mod.rs
@@ -2,18 +2,20 @@
mod group;
mod poll;
+mod uncommitted;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
-use common_kafka_consumer::{Charge, Offset, TopicOffsetLedger, TopicPartition};
+use common_kafka_consumer::{Charge, Held, Offset, Rejection, TopicOffsetLedger, TopicPartition};
use crate::config::CompletionGranularity;
use crate::order_sentinel::OffsetSpan;
pub(crate) use group::GroupSettlement;
pub(crate) use poll::PollSettlement;
+use uncommitted::Uncommitted;
pub(crate) struct ChargedSlice<'a> {
pub(crate) topic_partition: &'a TopicPartition,
@@ -23,6 +25,13 @@ pub(crate) struct ChargedSlice<'a> {
}
pub(crate) trait Settlement: Send + Sync {
+ fn charge(
+ &self,
+ topic_partition: &TopicPartition,
+ generation: u64,
+ charges: &[(Offset, Charge)],
+ ) -> Result<Held, Rejection>;
+
/// Accepted offsets from one poll's slice of a partition, charged under `generation`.
fn settle_run(
&self,
diff --git a/rust/ingestion-consumer/src/settlement/poll.rs b/rust/ingestion-consumer/src/settlement/poll.rs
index 7a58b81c332e..1ac43e2a071f 100644
--- a/rust/ingestion-consumer/src/settlement/poll.rs
+++ b/rust/ingestion-consumer/src/settlement/poll.rs
@@ -2,29 +2,37 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
-use common_kafka_consumer::{Offset, TopicOffsetLedger, TopicPartition};
+use common_kafka_consumer::{Charge, Held, Offset, Rejection, TopicOffsetLedger, TopicPartition};
use metrics::counter;
use tracing::warn;
-use super::{ChargedSlice, Settlement};
+use super::{ChargedSlice, Settlement, Uncommitted};
use crate::commit_pacer::CommitPacer;
use crate::ledger_rejection::{report_rejection, RejectedSlice};
pub(crate) struct PollSettlement {
- ledger: Arc<TopicOffsetLedger>,
- pacer: Arc<CommitPacer>,
+ uncommitted: Uncommitted,
}
impl PollSettlement {
pub(crate) fn new(ledger: Arc<TopicOffsetLedger>) -> Self {
Self {
- ledger,
- pacer: Arc::new(CommitPacer::immediate()),
+ uncommitted: Uncommitted::new(ledger, CommitPacer::immediate()),
}
}
}
impl Settlement for PollSettlement {
+ fn charge(
+ &self,
+ topic_partition: &TopicPartition,
+ generation: u64,
+ charges: &[(Offset, Charge)],
+ ) -> Result<Held, Rejection> {
+ self.uncommitted
+ .charge(topic_partition, generation, charges)
+ }
+
fn settle_run(
&self,
_topic_partition: &TopicPartition,
@@ -43,7 +51,7 @@ impl Settlement for PollSettlement {
return Ok(());
}
- if let Some(no_frontier) = settle_slices(&self.ledger, &self.pacer, slices)? {
+ if let Some(no_frontier) = settle_slices(&self.uncommitted, slices)? {
let reason = no_frontier.reason();
counter!("ingestion_consumer_commits_skipped_total", "reason" => reason).increment(1);
warn!(reason, "Poll handed over no frontier; nothing to commit");
@@ -52,15 +60,15 @@ impl Settlement for PollSettlement {
}
fn take_due(&self, now: Instant) -> Option<HashMap<TopicPartition, Offset>> {
- self.pacer.take_due(now)
+ self.uncommitted.take_due(now)
}
fn take_all(&self) -> Option<HashMap<TopicPartition, Offset>> {
- self.pacer.take_all()
+ self.uncommitted.take_all()
}
fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
- self.pacer.forget_partitions(topic_partitions);
+ self.uncommitted.forget_partitions(topic_partitions);
}
}
@@ -79,14 +87,13 @@ impl NoFrontier {
}
fn settle_slices(
- ledger: &TopicOffsetLedger,
- pacer: &CommitPacer,
+ uncommitted: &Uncommitted,
slices: &[ChargedSlice<'_>],
) -> anyhow::Result<Option<NoFrontier>> {
let mut settled = false;
let mut handed_over = false;
for slice in slices {
- let frontier = match ledger.settle(
+ let frontier = match uncommitted.settled(
slice.topic_partition,
slice.generation,
slice.charges.iter().map(|(offset, _)| *offset),
@@ -103,13 +110,7 @@ fn settle_slices(
}
};
settled = true;
- if frontier.is_none() {
- continue;
- }
- if let Some(taken) = ledger.take_frontier(slice.topic_partition) {
- pacer.advance_frontier(slice.topic_partition, taken);
- handed_over = true;
- }
+ handed_over |= frontier.is_some();
}
Ok(match (handed_over, settled) {
(true, _) => None,
diff --git a/rust/ingestion-consumer/src/settlement/uncommitted.rs b/rust/ingestion-consumer/src/settlement/uncommitted.rs
new file mode 100644
index 000000000000..c6c62a0ecab2
--- /dev/null
+++ b/rust/ingestion-consumer/src/settlement/uncommitted.rs
@@ -0,0 +1,288 @@
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+
+use common_kafka_consumer::{Charge, Held, Offset, Rejection, TopicOffsetLedger, TopicPartition};
+use metrics::gauge;
+
+use crate::commit_pacer::CommitPacer;
+
+const UNCOMMITTED_OFFSETS: &str = "kafka_consumer_ledger_uncommitted_offsets";
+const UNCOMMITTED_EVENTS: &str = "kafka_consumer_ledger_uncommitted_events";
+const UNCOMMITTED_BYTES: &str = "kafka_consumer_ledger_uncommitted_bytes";
+
+/// Counts each partition's charge from poll until the consumer makes the Kafka
+/// commit call that covers it: first in the ledger's window, then in the pacer.
+/// That call is asynchronous, and nothing here waits for Kafka to confirm it.
+///
+/// Every call runs on the consumer loop task, one at a time, so a move between
+/// the ledger and the pacer never interleaves with a read of both.
+pub(super) struct Uncommitted {
+ ledger: Arc<TopicOffsetLedger>,
+ pacer: Mutex<CommitPacer>,
+}
+
+impl Uncommitted {
+ pub(super) fn new(ledger: Arc<TopicOffsetLedger>, pacer: CommitPacer) -> Self {
+ Self {
+ ledger,
+ pacer: Mutex::new(pacer),
+ }
+ }
+
+ pub(super) fn charge(
+ &self,
+ topic_partition: &TopicPartition,
+ generation: u64,
+ charges: &[(Offset, Charge)],
+ ) -> Result<Held, Rejection> {
+ let held = self
+ .ledger
+ .charge(topic_partition, generation, charges.iter().copied())?;
+ let uncommitted = held + self.pacer.lock().unwrap().pending(topic_partition);
+ publish(topic_partition, uncommitted);
+ Ok(uncommitted)
+ }
+
+ /// Called after a group run or a poll settles: mark its offsets complete
+ /// and move the frontier that results, with its charge, to the pacer.
+ /// Returns that frontier, `None` while the front of the window is still
+ /// incomplete.
+ pub(super) fn settled(
+ &self,
+ topic_partition: &TopicPartition,
+ generation: u64,
+ offsets: impl IntoIterator<Item = Offset>,
+ ) -> Result<Option<Offset>, Rejection> {
+ let frontier = self.ledger.settle(topic_partition, generation, offsets)?;
+ if frontier.is_some() {
+ if let Some(taken) = self.ledger.take_frontier(topic_partition) {
+ self.pacer
+ .lock()
+ .unwrap()
+ .advance_frontier(topic_partition, taken);
+ }
+ }
+ Ok(frontier)
+ }
+
+ /// Refunds the charge of every frontier handed out; the consumer submits
+ /// their commit right after.
+ pub(super) fn take_due(&self, now: Instant) -> Option<HashMap<TopicPartition, Offset>> {
+ let due = self.pacer.lock().unwrap().take_due(now)?;
+ self.publish_window(&due);
+ Some(due)
+ }
+
+ /// Like `take_due`, without waiting for the interval.
+ pub(super) fn take_all(&self) -> Option<HashMap<TopicPartition, Offset>> {
+ let due = self.pacer.lock().unwrap().take_all()?;
+ self.publish_window(&due);
+ Some(due)
+ }
+
+ pub(super) fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
+ self.ledger.forget_partitions(
+ topic_partitions
+ .iter()
+ .map(|tp| (tp.topic.as_str(), tp.partition)),
+ );
+ self.pacer
+ .lock()
+ .unwrap()
+ .forget_partitions(topic_partitions);
+ for topic_partition in topic_partitions {
+ publish(topic_partition, Held::default());
+ }
+ }
+
+ fn publish_window(&self, due: &HashMap<TopicPartition, Offset>) {
+ for topic_partition in due.keys() {
+ publish(topic_partition, self.ledger.held(topic_partition));
+ }
+ }
+}
+
+fn publish(topic_partition: &TopicPartition, uncommitted: Held) {
+ let topic: Arc<str> = Arc::from(topic_partition.topic.as_str());
+ let partition: Arc<str> = Arc::from(topic_partition.partition.to_string());
+ gauge!(
+ UNCOMMITTED_OFFSETS,
+ "topic" => topic.clone(),
+ "partition" => partition.clone()
+ )
+ .set(uncommitted.offsets as f64);
+ gauge!(
+ UNCOMMITTED_EVENTS,
+ "topic" => topic.clone(),
+ "partition" => partition.clone()
+ )
+ .set(uncommitted.charge.events as f64);
+ gauge!(
+ UNCOMMITTED_BYTES,
+ "topic" => topic,
+ "partition" => partition
+ )
+ .set(uncommitted.charge.bytes as f64);
+}
+
+#[cfg(test)]
+mod tests {
+ use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
+
+ use super::*;
+
+ const EACH: Charge = Charge {
+ events: 1,
+ bytes: 100,
+ };
+
+ fn p0() -> TopicPartition {
+ TopicPartition::new("events", 0)
+ }
+
+ fn held(offsets: usize) -> Held {
+ Held {
+ offsets,
+ charge: Charge {
+ events: offsets as u64,
+ bytes: 100 * offsets as u64,
+ },
+ }
+ }
+
+ fn uncommitted() -> Uncommitted {
+ Uncommitted::new(Arc::new(TopicOffsetLedger::new()), CommitPacer::immediate())
+ }
+
+ fn published(snapshotter: &Snapshotter) -> Held {
+ let mut published = Held::default();
+ for (key, _, _, value) in snapshotter.snapshot().into_vec() {
+ let DebugValue::Gauge(value) = value else {
+ continue;
+ };
+ let value = value.into_inner();
+ match key.key().name() {
+ UNCOMMITTED_OFFSETS => published.offsets = value as usize,
+ UNCOMMITTED_EVENTS => published.charge.events = value as u64,
+ UNCOMMITTED_BYTES => published.charge.bytes = value as u64,
+ _ => {}
+ }
+ }
+ published
+ }
+
+ fn charge(uncommitted: &Uncommitted, generation: u64, offset: i64) -> Held {
+ uncommitted
+ .charge(&p0(), generation, &[(Offset(offset), EACH)])
+ .expect("the ledger accepts the charge")
+ }
+
+ fn settled(uncommitted: &Uncommitted, offsets: &[i64]) {
+ let frontier = uncommitted
+ .settled(&p0(), 0, offsets.iter().map(|&o| Offset(o)))
+ .expect("the ledger settles");
+ assert!(frontier.is_some());
+ }
+
+ #[test]
+ fn a_take_stays_uncommitted_until_the_pacer_hands_it_out() {
+ let uncommitted = uncommitted();
+ charge(&uncommitted, 0, 10);
+ settled(&uncommitted, &[10]);
+
+ assert_eq!(charge(&uncommitted, 0, 11), held(2));
+
+ uncommitted.take_due(Instant::now()).expect("due");
+ assert_eq!(charge(&uncommitted, 0, 12), held(2));
+ }
+
+ #[test]
+ fn takes_since_the_last_commit_add_up() {
+ let uncommitted = uncommitted();
+ charge(&uncommitted, 0, 10);
+ charge(&uncommitted, 0, 11);
+ settled(&uncommitted, &[10]);
+ settled(&uncommitted, &[11]);
+
+ assert_eq!(charge(&uncommitted, 0, 12), held(3));
+ }
+
+ #[test]
+ fn a_take_moves_only_its_own_partitions_charge() {
+ let uncommitted = uncommitted();
+ let p1 = TopicPartition::new("events", 1);
+ let charge_p1 = |offset: i64| {
+ uncommitted
+ .charge(&p1, 0, &[(Offset(offset), EACH)])
+ .expect("the ledger accepts the charge")
+ };
+ charge(&uncommitted, 0, 10);
+ charge_p1(20);
+ settled(&uncommitted, &[10]);
+
+ assert_eq!(charge_p1(21), held(2), "partition 0's take stays its own");
+
+ uncommitted.take_due(Instant::now()).expect("due");
+ assert_eq!(charge(&uncommitted, 0, 11), held(1));
+ assert_eq!(charge_p1(22), held(3));
+ }
+
+ #[test]
+ fn a_forgotten_partition_drops_its_window_and_its_pending_take() {
+ let uncommitted = uncommitted();
+ charge(&uncommitted, 0, 10);
+ charge(&uncommitted, 0, 11);
+ settled(&uncommitted, &[10]);
+
+ uncommitted.forget_partitions(&[p0()]);
+
+ assert!(uncommitted.take_all().is_none());
+ assert_eq!(charge(&uncommitted, 1, 10), held(1));
+ }
+
+ #[test]
+ fn a_take_over_a_gap_moves_the_gap_offsets_to_the_pacer() {
+ let uncommitted = uncommitted();
+ charge(&uncommitted, 0, 10);
+ let window = charge(&uncommitted, 0, 12);
+ assert_eq!(
+ window,
+ Held {
+ offsets: 3,
+ charge: EACH + EACH
+ }
+ );
+
+ settled(&uncommitted, &[10, 12]);
+
+ assert_eq!(
+ charge(&uncommitted, 0, 13),
+ Held {
+ offsets: 4,
+ charge: EACH + EACH + EACH
+ }
+ );
+ }
+
+ #[test]
+ fn the_gauges_follow_the_charge_until_the_frontier_is_handed_out() {
+ let recorder = DebuggingRecorder::new();
+ let snapshotter = recorder.snapshotter();
+ let _guard = metrics::set_default_local_recorder(&recorder);
+ let uncommitted = uncommitted();
+
+ charge(&uncommitted, 0, 10);
+ charge(&uncommitted, 0, 11);
+ assert_eq!(published(&snapshotter), held(2));
+
+ settled(&uncommitted, &[10]);
+ assert_eq!(published(&snapshotter), held(2), "a take moves the charge");
+
+ uncommitted.take_due(Instant::now()).expect("due");
+ assert_eq!(published(&snapshotter), held(1));
+
+ uncommitted.forget_partitions(&[p0()]);
+ assert_eq!(published(&snapshotter), Held::default());
+ }
+}
diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rs
index 175eb7f8b311..6378827308b9 100644
--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs
+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs
@@ -8,13 +8,14 @@ mod common;
use std::collections::HashSet;
use std::num::{NonZeroU32, NonZeroUsize};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use axum::routing::get;
use axum::Router;
use futures::StreamExt;
use lifecycle::{ComponentOptions, Manager};
+use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
use rdkafka::config::ClientConfig;
@@ -2617,7 +2618,8 @@ async fn group_granularity_commits_the_ledger_frontier() {
}
#[tokio::test]
-async fn group_granularity_commits_the_pending_frontier_on_stop() {
+async fn group_granularity_keeps_a_held_frontier_uncommitted_until_the_stop_commits_it() {
+ let metrics = metrics();
let topic = format!("e2e-group-stop-commit-{}", Uuid::new_v4());
let harness = Harness::start_group_granularity_committing_every(
common::PRODUCTION_PACK_TARGETS,
@@ -2651,6 +2653,11 @@ async fn group_granularity_commits_the_pending_frontier_on_stop() {
Some(1),
"the interval holds the later frontier"
);
+ assert_eq!(
+ uncommitted_offsets(metrics, &topic, 0),
+ Some(9.0),
+ "the held frontier is still uncommitted"
+ );
let group_id = harness.group_id.clone();
harness.stop().await;
@@ -2659,6 +2666,43 @@ async fn group_granularity_commits_the_pending_frontier_on_stop() {
broker_committed_offset(&group_id, &topic, 0) == Some(10)
})
.await;
+ assert_eq!(uncommitted_offsets(metrics, &topic, 0), Some(0.0));
+}
+
+// One recorder per test binary; each test reads only its own topic's series.
+fn metrics() -> &'static Snapshotter {
+ static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();
+ SNAPSHOTTER.get_or_init(|| {
+ let recorder = DebuggingRecorder::new();
+ let snapshotter = recorder.snapshotter();
+ metrics::set_global_recorder(recorder).expect("no other recorder in the e2e binary");
+ snapshotter
+ })
+}
+
+fn uncommitted_offsets(metrics: &Snapshotter, topic: &str, partition: i32) -> Option<f64> {
+ let partition = partition.to_string();
+ metrics
+ .snapshot()
+ .into_vec()
+ .into_iter()
+ .find_map(|(key, _, _, value)| {
+ let key = key.key();
+ let labeled = |name: &str, expected: &str| {
+ key.labels()
+ .any(|label| label.key() == name && label.value() == expected)
+ };
+ if key.name() != "kafka_consumer_ledger_uncommitted_offsets"
+ || !labeled("topic", topic)
+ || !labeled("partition", &partition)
+ {
+ return None;
+ }
+ match value {
+ DebugValue::Gauge(value) => Some(value.into_inner()),
+ _ => None,
+ }
+ })
}
const STALL_KEYS: [(i32, &str); 2] = [(0, "user-a"), (1, "user-b")];