:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion-consumer): poll kafka continuously behind a polling strategy (#121206)
Paweł Ledwoń19d7605ca0a8
diff --git a/rust/common/kafka-consumer/src/accumulator.rs b/rust/common/kafka-consumer/src/accumulator.rsindex 509cf2002ecc..ed82e4cf2a94 100644--- a/rust/common/kafka-consumer/src/accumulator.rs+++ b/rust/common/kafka-consumer/src/accumulator.rs@@ -42,8 +42,7 @@ impl<K, M> Group<K, M> {/// Demuxes one poll into groups. Push messages in delivery order: a keyed/// message joins its partition's group for that key, a keyless message opens-/// a group of its own. Groups come out in first-seen order, and each group-/// keeps its messages in submission order.+/// a group of its own. Groups come out in first-seen order.#[derive(Debug)]pub struct Accumulator<K, M> {groups: Vec<Group<K, M>>,@@ -103,6 +102,30 @@ impl<K: Hash + Eq + Clone, M> Accumulator<K, M> {self.message_count}+ /// Returns how many messages it removed.+ pub fn remove_partition(&mut self, partition: Partition) -> usize {+ let before = self.message_count;+ self.groups.retain(|group| {+ if group.partition != partition {+ return true;+ }+ self.message_count -= group.messages.len();+ false+ });+ if before != self.message_count {+ self.group_index_by_key.clear();+ for (index, group) in self.groups.iter().enumerate() {+ if let Some(key) = &group.key {+ self.group_index_by_key+ .entry(group.partition)+ .or_default()+ .insert(key.clone(), index);+ }+ }+ }+ before - self.message_count+ }+pub fn into_groups(self) -> Vec<Group<K, M>> {self.groups}@@ -158,6 +181,34 @@ mod tests {assert_eq!(bodies, vec![0, 2]);}+ #[test]+ fn a_removed_partition_leaves_the_other_groups_open_for_their_keys() {+ let mut acc = Accumulator::default();+ push(&mut acc, 0, 0, Some("a"));+ push(&mut acc, 1, 5, Some("a"));+ push(&mut acc, 0, 1, None);+ push(&mut acc, 1, 6, Some("b"));++ assert_eq!(acc.remove_partition(Partition(0)), 2);+ push(&mut acc, 1, 7, Some("a"));+ push(&mut acc, 0, 2, Some("a"));++ assert_eq!(acc.message_count(), 4);+ let shapes: Vec<(Partition, Option<&str>, Vec<Offset>)> = acc+ .into_groups()+ .iter()+ .map(|g| (g.partition, g.key, offsets(g)))+ .collect();+ assert_eq!(+ shapes,+ vec![+ (Partition(1), Some("a"), vec![Offset(5), Offset(7)]),+ (Partition(1), Some("b"), vec![Offset(6)]),+ (Partition(0), Some("a"), vec![Offset(2)]),+ ]+ );+ }+#[test]fn keyless_messages_each_form_their_own_group() {let mut acc = Accumulator::default();diff --git a/rust/common/kafka-consumer/src/topic_offset_ledger.rs b/rust/common/kafka-consumer/src/topic_offset_ledger.rsindex 435da74605e0..0a8c368e419f 100644--- a/rust/common/kafka-consumer/src/topic_offset_ledger.rs+++ b/rust/common/kafka-consumer/src/topic_offset_ledger.rs@@ -1,5 +1,4 @@use std::collections::HashMap;-use std::sync::atomic::{AtomicU64, Ordering};use std::sync::{Arc, Mutex};use metrics::counter;@@ -9,8 +8,8 @@ use crate::charge::Charge;use crate::partition_offset_ledger::{Held, LedgerError, PartitionOffsetLedger, TakenFrontier};use crate::types::Offset;-/// Charges and settlements dropped because their partition was reassigned-/// while they were in flight.+/// Charges and settlements dropped because their stamp predates the+/// partition's current generation: a reassignment or a violation reset.const STALE_SLICES: &str = "kafka_consumer_ledger_stale_slices_total";/// Contract violations in the ledger's accounting; must stay at zero.const ERRORS: &str = "kafka_consumer_ledger_errors_total";@@ -36,8 +35,9 @@ impl TopicPartition {/// Why a charge or settlement did not land on its partition's ledger.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum Rejection {- /// The work predates the partition's current assignment. Kafka- /// redelivers its offsets under the current one, so nothing is lost.+ /// The work's stamp predates the partition's current generation: the+ /// partition was reassigned, or its ledger was reset after a violation.+ /// After a reassignment Kafka redelivers the offsets, so nothing is lost.Stale { stamp: u64, generation: u64 },/// The work violated the ledger's contract. The partition's window is/// unknown after this, so its ledger was reset to a new generation: work@@ -64,11 +64,6 @@ pub enum Rejection {#[derive(Default)]pub struct TopicOffsetLedger {partitions: Mutex<HashMap<TopicPartition, PartitionOffsetLedger>>,- /// Version of the per-partition generation map: moves whenever any- /// partition's generation does. A caller stamping work outside the lock- /// reads this per message and only re-reads the partition's generation- /// when it moved.- generations_version: AtomicU64,}impl TopicOffsetLedger {@@ -87,13 +82,6 @@ impl TopicOffsetLedger {.unwrap_or_default()}- /// Version of the per-partition generation map. Cheaper than- /// `generation` and moves whenever any generation does, so a caller- /// compares it per message and consults `generation` only on a change.- pub fn generations_version(&self) -> u64 {- self.generations_version.load(Ordering::Relaxed)- }-/// Record one slice of delivered offsets on the partition's ledger,/// founding the ledger when the slice is the partition's first delivery./// Returns what the partition's window holds after the charge, or why@@ -118,7 +106,6 @@ impl TopicOffsetLedger {Err(error) => {let held = ledger.held();*ledger = PartitionOffsetLedger::new(generation + 1);- self.generations_version.fetch_add(1, Ordering::Relaxed);Err(Rejection::Violation {error,stamp,@@ -164,7 +151,6 @@ impl TopicOffsetLedger {Err(error) => {let held = ledger.held();*ledger = PartitionOffsetLedger::new(generation + 1);- self.generations_version.fetch_add(1, Ordering::Relaxed);Err(Rejection::Violation {error,stamp,@@ -218,7 +204,6 @@ impl TopicOffsetLedger {topic_partitions: impl IntoIterator<Item = (&'a str, i32)>,) {let mut partitions = self.partitions.lock().unwrap();- let mut forgotten = 0;for (topic, partition) in topic_partitions {let key = TopicPartition::new(topic, partition);let generation = partitions@@ -226,10 +211,7 @@ impl TopicOffsetLedger {.map(PartitionOffsetLedger::generation).unwrap_or_default();partitions.insert(key, PartitionOffsetLedger::new(generation + 1));- forgotten += 1;}- self.generations_version- .fetch_add(forgotten, Ordering::Relaxed);}}@@ -265,7 +247,6 @@ fn count_rejection(stage: &'static str, rejection: Rejection) {}}-/// One partition's label values, built once per emission.fn labels(topic_partition: &TopicPartition) -> (Arc<str>, Arc<str>) {(Arc::from(topic_partition.topic.as_str()),@@ -315,19 +296,15 @@ mod tests {let ledger = TopicOffsetLedger::new();let p0 = tp("events", 0);charge(&ledger, &p0, 0, &[10]);- assert_eq!(ledger.generations_version(), 0);ledger.forget_partitions([("events", 0)]);assert_eq!(ledger.held(&p0).offsets, 0);assert_eq!(ledger.generation(&p0), 1);- assert_eq!(ledger.generations_version(), 1);}#[test]fn forgetting_an_unseen_partition_still_opens_a_generation() {- // A slice buffered before the revoke carries stamp 0; without a new- // generation it would found the new assignment's ledger with stale offsets.let ledger = TopicOffsetLedger::new();let p0 = tp("events", 0);ledger.forget_partitions([("events", 0)]);@@ -362,8 +339,6 @@ mod tests {assert!(ledger.take_frontier(&p0).is_none());assert_eq!(ledger.held(&p0).offsets, 2);- // The late completion arrives with the next batch and the held- // offsets drain.let frontier = ledger.settle(&p0, 0, [Offset(10)]).expect("live ledger settles");@@ -453,8 +428,7 @@ mod tests {let p0 = tp("events", 0);charge(&ledger, &p0, 0, &[10]);- // The partition is revoked and reassigned to this consumer while the- // batch is still in flight; the redelivery recharges the same offset.+ // Revoke and reassign while the batch is in flight.ledger.forget_partitions([("events", 0)]);charge(&ledger, &p0, 1, &[10]);@@ -471,7 +445,6 @@ mod tests {"the redelivered offset stays uncompleted");- // The redelivery's own completion settles the new assignment's ledger.ledger.settle(&p0, 1, [Offset(10)]).expect("current-generation batch settles");@@ -487,7 +460,6 @@ mod tests {charge(&ledger, &p0, 0, &[10]);charge(&ledger, &p1, 0, &[20]);- // Only partition 0 is revoked and reassigned.ledger.forget_partitions([("events", 0)]);charge(&ledger, &p0, 1, &[10]);assert_eq!(@@ -526,10 +498,7 @@ mod tests {let p1 = tp("events", 1);charge(&ledger, &p1, 0, &[20]);- // Another partition's reassignment moves the generations version;- // this partition's generation and ledger are untouched.ledger.forget_partitions([("events", 0)]);- assert_eq!(ledger.generations_version(), 1);charge(&ledger, &p1, 0, &[21]);assert_eq!(ledger.held(&p1).offsets, 2);@@ -546,8 +515,7 @@ mod tests {let p0 = tp("events", 0);charge(&ledger, &p0, 0, &[10]);- // The partition leaves for another consumer and returns later; both- // callbacks forget it, and the in-flight batch settles in between.+ // The revoke and the later assign both forget the partition.ledger.forget_partitions([("events", 0)]);ledger.forget_partitions([("events", 0)]);assert_eq!(ledger.generation(&p0), 2);@@ -607,8 +575,6 @@ mod tests {let p0 = tp("events", 0);charge(&ledger, &p0, 0, &[10, 11]);- // A duplicate delivery under the same generation is a contract- // violation, not a rebalance: the ledger cannot trust its window.assert_eq!(ledger.charge(&p0, 0, [(Offset(11), Charge::ZERO)]),Err(Rejection::Violation {@@ -626,10 +592,7 @@ mod tests {);assert_eq!(ledger.held(&p0).offsets, 0, "the window is discarded");assert_eq!(ledger.generation(&p0), 1);- assert_eq!(ledger.generations_version(), 1);- // In-flight work from before the reset drops as stale; the next- // delivery under the new generation founds a fresh ledger.assert_eq!(ledger.settle(&p0, 0, [Offset(10)]),Err(Rejection::Stale {diff --git a/rust/common/kafka-consumer/src/types.rs b/rust/common/kafka-consumer/src/types.rsindex a656efe5756e..76a11a6a7353 100644--- a/rust/common/kafka-consumer/src/types.rs+++ b/rust/common/kafka-consumer/src/types.rs@@ -12,24 +12,26 @@ impl fmt::Display for Partition {}}-/// One group's completion: the group's messages were delivered downstream and-/// the consumer may account them toward its poll and, later, its commit. No-/// batch identity and no message bodies cross this boundary; the consumer-/// correlates by partition and offset.+/// Accepted offsets of one partition and generation from one worker+/// response: the messages were delivered downstream and the consumer may+/// account them toward its poll and, later, its commit. No batch identity+/// and no message bodies cross this boundary; the consumer correlates by+/// partition and offset.#[derive(Debug, Clone, PartialEq, Eq)]pub struct GroupCompletion {pub partition: Partition,- /// The assignment epoch under which the group's poll was collected. A- /// completion for a partition that was revoked and reassigned in between- /// carries an older epoch than any poll of the new incarnation.+ /// The assignment epoch the messages' poll was submitted under. A+ /// completion for a partition that was revoked and reassigned since then+ /// carries an older epoch than any poll after the reassignment.pub assignment_epoch: u64,- /// The offsets of the group's messages, in group order.+ /// The ledger generation the group's messages were charged under.+ pub generation: u64,+ /// The accepted offsets, in send order.pub offsets: Vec<Offset>,- /// How many of the group's messages the worker accepted.+ /// Always `offsets.len()`: a completion carries accepted offsets only.pub accepted: u32,}-/// A Kafka message offset.#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct Offset(pub i64);diff --git a/rust/ingestion-consumer/README.md b/rust/ingestion-consumer/README.mdindex 200439867df6..a1223b3d3ffa 100644--- a/rust/ingestion-consumer/README.md+++ b/rust/ingestion-consumer/README.md@@ -42,7 +42,7 @@ A partition that leaves the assignment drops its pending frontier with its ledge`group` settles each group's offsets as its completion arrives, in any order, so a stalled key holds back only its own partition's commit.At `group` granularity the pacer keeps the latest frontier per partition and hands them out at most once per `CONSUMER_COMMIT_INTERVAL_MS` (default 500 ms).The consumer asks the pacer on each completion wake, at the top of each loop turn, and on the completion wait's one-second heartbeat, and commits everything pending on exit.-Admission is still per poll at both granularities: `CONSUMER_MAX_BACKGROUND_TASKS` caps in-flight polls, and a slot frees when its whole poll is covered, oldest first, so a stalled key still holds its poll's slot.+Under the capped polling strategy, admission is per poll at both granularities: `CONSUMER_MAX_BACKGROUND_TASKS` caps in-flight polls, and a slot frees when its whole poll is covered, oldest first, so a stalled key still holds its poll's slot. See [Polling](#polling).At `poll` granularity, `ingestion_consumer_commits_skipped_total{reason}` counts polls that settle with no frontier: `rejected` means the ledger dropped every slice (expected around a rebalance) and `no_frontier` means an earlier batch is still incomplete at the front of every window the batch settled.The settlement publishes the uncommitted gauges, because it holds both places a partition's charge can be: the ledger's window and the pacer.`kafka_consumer_ledger_uncommitted_offsets{topic,partition}` gauges the offsets each partition has polled and no commit has covered yet, including those taken for a commit the pacer still holds; `kafka_consumer_ledger_uncommitted_events` and `kafka_consumer_ledger_uncommitted_bytes` gauge the charge those offsets carry, where bytes is the payload plus key plus headers of each message.@@ -56,12 +56,33 @@ The ledger emits its own counters, so any consumer built on the crate reports thA budget needs `CONSUMER_COMPLETION_GRANULARITY=group`. At `poll` a stalled partition holds back the commits of every partition in its poll, so their budgets would fill and pause together. The consumer ignores a budget at `poll` and logs a warning, so switching a lane back to `poll` rolls it back without unsetting the budget.When either axis reaches its limit after a poll, the consumer pauses that partition's fetch through the Kafka client; when a commit brings both axes back under `CONSUMER_PARTITION_BUDGET_LOW_WATERMARK` (a fraction of the budget, default 0.8), it resumes.The hysteresis matters because a pause purges the partition's fetched messages from librdkafka's queue and a resume refetches them.-A pause stops only the partition's fetch: the consumer keeps polling, so rebalance callbacks still run and `max.poll.interval.ms` stays satisfied, except while the loop waits on its oldest in-flight poll.+A pause stops only the partition's fetch: the consumer keeps polling, so rebalance callbacks still run and `max.poll.interval.ms` stays satisfied, except while the capped polling strategy's cap is full or the continuous strategy is at its in-flight limit.A revoked partition is forgotten. librdkafka keeps a partition's pause across a revoke and a reassignment, so the consumer resumes every partition it is assigned, and the partition starts with a fresh ledger.Both limits default to `0`, which sets no budget; the admission cap (`CONSUMER_MAX_BACKGROUND_TASKS`) then governs alone.The budget is a fixed amount per partition, not a share of a pod-wide amount, so a rebalance changes how many partitions a pod holds, not how much each may expose.`ingestion_consumer_budget_paused_partitions` gauges how many partitions are paused and `ingestion_consumer_budget_pauses_total{topic,partition}` counts pauses; read them against `ingestion_lag_ms`.+## Polling++The consumer loop is the only caller of `recv()`. It waits on Kafka, completions, the commit tick, the shutdown signal, and the poll's batch timeout, and hands each event to the selected implementation of the `PollingStrategy` trait in `src/polling/`.+The strategy returns steps, and the consumer carries them out: charge and submit a poll, submit a commit and resume its partitions, stop, or fail.+A poll is submitted when it reaches the batch size or the byte bound, or when `CONSUMER_BATCH_TIMEOUT_MS` passes after its first message.++The rebalance callbacks run inside `recv()`, on the loop's task. After every wake the loop hands the partitions they revoked to the strategy, before the event that woke it.+The strategy then removes those partitions' messages from the poll in progress and from its in-flight state: the batcher purges them, or Kafka delivers them again from the last commit.+So a message the loop adds to a poll always belongs to the current assignment, and the poll is charged under the partition's current ledger generation.++`CONSUMER_POLLING_STRATEGY` selects what bounds admission.+`capped` (the default) admits up to `CONSUMER_MAX_BACKGROUND_TASKS` polls, and frees a slot only when the oldest poll is covered, so a stalled key holds back fetching for every partition.++`continuous` admits polls without waiting on any of them.+The partition budget bounds each partition's replay exposure, and the in-flight limit bounds the whole pod: `CONSUMER_IN_FLIGHT_EVENTS` and `CONSUMER_IN_FLIGHT_BYTES` count the uncommitted charge of every partition, as the partition budget counts it.+The limit is approximate: it stops reading, and the poll in progress still completes, so the uncommitted charge can pass the limit by one poll. One poll holds at most `CONSUMER_BATCH_SIZE` events and `CONSUMER_BATCH_SIZE_KB` plus one message; with the byte bound off, its bytes are bounded only by the batch size times the largest message.+Room frees when a commit hands out offsets, as it does for the partition budget, not when a completion arrives.+At the limit the loop stops reading Kafka until a commit frees room, and rebalance callbacks wait with it, so size the limit above the budget times the partitions a pod holds.+`continuous` needs `CONSUMER_COMPLETION_GRANULARITY=group`, a partition budget, and an in-flight limit, and the budget and the limit each need both their events and bytes settings. A message with no key, value or headers charges zero bytes, so a byte-only limit admits any number of them. Without all three the consumer logs a warning and runs `capped`, so switching a lane back to `poll` rolls back both.+`ingestion_consumer_in_flight_limit_reached_total` counts each time the loop stops reading at the limit; the `kafka_consumer_ledger_uncommitted_*` gauges show each partition's share.+## BatcherThe batcher state machine in `src/batcher/` orders runs and places them on workers.@@ -145,7 +166,7 @@ Remaining: alert rules on the ledger counters and on kminion lag.A message a worker permanently rejects (a nack on every delivery) has no dead-letter path: after the stall timeout (`CONSUMER_DEFERRED_FLUSH_TIMEOUT_MS`) the consumer fails and the process crash-loops on redelivery.`poison_batch_fails_safely_without_committing` pins the safe half of the trade-off (never commit past unaccepted messages).Needs a design pass: DLQ topic, retry budget before giving up, the per-key ordering caveat (DLQ-ing one message then delivering later ones for the same person), and replay ownership.-Non-UTF-8 payloads — currently nulled in `collect_batch` with no metric — should route to the same DLQ as malformed input; a bytes-preserving wire format is not worth it while the producer (capture) guarantees UTF-8 JSON.+Non-UTF-8 payloads — currently nulled when a poll adds the message, with no metric — should route to the same DLQ as malformed input; a bytes-preserving wire format is not worth it while the producer (capture) guarantees UTF-8 JSON.### 4. Revoke-aware partition handoffdiff --git a/rust/ingestion-consumer/docs/driver-model-plan.md b/rust/ingestion-consumer/docs/driver-model-plan.mdindex 826d098c3ccc..e80394c8cf91 100644--- a/rust/ingestion-consumer/docs/driver-model-plan.md+++ b/rust/ingestion-consumer/docs/driver-model-plan.md@@ -101,11 +101,14 @@ A change that adds metrics lists their names.| 22 | 8 decoupled commits | Cleanup | Delete the per-poll completion path || 23 | 9 bounded replay | Implement | Add the budget accounting || 24 | 9 bounded replay | Switchover | Enable the budget |-| 25 | 9 bounded replay | Structure | Poll continuously and remove the in-flight admission cap |-| 26 | 10 clean handoff | Prepare | Add revoke and drained markers |-| 27 | 10 clean handoff | Implement | Build the drain |-| 28 | 10 clean handoff | Switchover | Enable the drain |-| 29 | 10 clean handoff | Cleanup | Delete the no-drain path |+| 25 | 9 bounded replay | Prepare | Put polling behind a strategy |+| 26 | 9 bounded replay | Implement | Build the continuous polling strategy |+| 27 | 9 bounded replay | Switchover | Poll continuously |+| 28 | 9 bounded replay | Cleanup | Delete the capped polling strategy |+| 29 | 10 clean handoff | Prepare | Add revoke and drained markers |+| 30 | 10 clean handoff | Implement | Build the drain |+| 31 | 10 clean handoff | Switchover | Enable the drain |+| 32 | 10 clean handoff | Cleanup | Delete the no-drain path |## Cycle 1: one send origin@@ -584,7 +587,7 @@ Exit criterion at the canary: zero ledger errors and gap walks, and a stalled pa- Keep `max_in_flight_batches` as the bound on outstanding work. A poll's slot frees when all its groups are accepted.- Behavior change: commit timing decouples across partitions and polls. Replay exposure stays inside the admission cap.- Rollback is the switch back to `poll`.-- Commit pacing is the seam for this cycle. The consumer already hands every taken frontier to a commit pacer (`commit_pacer.rs`) and commits what the pacer hands back. Today's pacer is `ImmediateCommitPacer`: everything handed over is due on the next take, so commits stay per poll. Per-partition commits at `group` granularity would commit on every group completion, which is too many commits, so this cycle gives the pacer an interval: it keeps the latest frontier per partition and hands them out at most once per `CONSUMER_COMMIT_INTERVAL_MS`, and a zero interval is today's immediate pacer. The loop then asks the pacer on a tick, not only after a poll settles, and the tick must not sit in a `select!` next to `collect_batch`, because a dropped collection loses messages before they reach the ledger; the interval bounds the commit delay only if the collection loop itself asks the pacer, otherwise the bound is the interval plus the batch timeout. `ingestion_consumer_committed_offset` is set when a commit is submitted to the client, so a frontier the pacer still holds is not reported as committed. On revoke the pacer forgets the partition; the drain in change 27 commits what the pacer still holds at shutdown. One `CommitPacer` with an interval covers both granularities; `poll` runs it at zero. The commit sentinel and the commit monitor retire here. The sentinel's contiguity check compared the span a poll delivered against the previous commit; at `group` granularity a take no longer maps to one poll's slice, and the ledger already rejects a redelivered offset at charge (`kafka_consumer_ledger_errors_total`) and counts the offsets a frontier walks over (`kafka_consumer_ledger_gap_offsets_total`). The monitor's broker comparison duplicated kminion's per-partition consumer group lag. Checking the take's own window base against the previous take is not a replacement: the ledger sets the base to the frontier it hands out, so that comparison holds by construction.+- Commit pacing is the seam for this cycle. The consumer already hands every taken frontier to a commit pacer (`commit_pacer.rs`) and commits what the pacer hands back. Today's pacer is `ImmediateCommitPacer`: everything handed over is due on the next take, so commits stay per poll. Per-partition commits at `group` granularity would commit on every group completion, which is too many commits, so this cycle gives the pacer an interval: it keeps the latest frontier per partition and hands them out at most once per `CONSUMER_COMMIT_INTERVAL_MS`, and a zero interval is today's immediate pacer. The loop then asks the pacer on a tick, not only after a poll settles, and the tick must not sit in a `select!` next to `collect_batch`, because a dropped collection loses messages before they reach the ledger; the interval bounds the commit delay only if the collection loop itself asks the pacer, otherwise the bound is the interval plus the batch timeout. `ingestion_consumer_committed_offset` is set when a commit is submitted to the client, so a frontier the pacer still holds is not reported as committed. On revoke the pacer forgets the partition; the drain in change 30 commits what the pacer still holds at shutdown. One `CommitPacer` with an interval covers both granularities; `poll` runs it at zero. The commit sentinel and the commit monitor retire here. The sentinel's contiguity check compared the span a poll delivered against the previous commit; at `group` granularity a take no longer maps to one poll's slice, and the ledger already rejects a redelivered offset at charge (`kafka_consumer_ledger_errors_total`) and counts the offsets a frontier walks over (`kafka_consumer_ledger_gap_offsets_total`). The monitor's broker comparison duplicated kminion's per-partition consumer group lag. Checking the take's own window base against the previous take is not a replacement: the ledger sets the base to the frontier it hands out, so that comparison holds by construction.**Interfaces:**@@ -602,11 +605,11 @@ Exit criterion at the canary: zero ledger errors and gap walks, and a stalled pa**Interfaces:**- Remove `PollSettlement` and the completion granularity from `Config`.-- Keep the in-flight poll queue and its per-poll aggregation from change 7. The loop waits on the oldest poll until change 25 replaces it.+- Keep the in-flight poll queue and its per-poll aggregation from change 7. The capped polling strategy waits on the oldest poll until change 28 deletes it.## Cycle 9: replay exposure is bounded per partition-Outcome: no partition's uncommitted work exceeds its budget, in events and bytes. The in-flight admission cap is gone. Needs cycles 2 and 8.+Outcome: no partition's uncommitted work exceeds its budget, in events and bytes. The poll-count admission cap is gone; a pod-wide in-flight limit bounds memory. Needs cycles 2 and 8.**Verify:** the ledger's uncommitted gauges run on all lanes with no budget set.Exit criterion: `kafka_consumer_ledger_uncommitted_events` summed per pod tracks the admission cap's occupancy.@@ -642,7 +645,7 @@ The budget is per partition, and so is the pause. One budget for the whole assig- A charts values PR sets the budget lane by lane. The admission cap and the budget both bound work during the transition.- Derive the first values from the current config: the cap times the batch size, divided by the partitions a pod holds, for events; the byte bound likewise for bytes. The per-pod bound then does not change at the switchover.- Pause a partition when its outstanding charge reaches the budget on either axis. Resume it only when its charge is back at or under the low watermark on both axes. A pause purges that partition's fetched messages from librdkafka's queue and a resume refetches them, so a gate that reopens on the first refund refetches on every crossing.-- Keep polling while some partitions are paused. A paused partition still serves rebalance callbacks and keeps `max.poll.interval.ms` alive; an unpolled consumer does neither. While every partition is paused, the loop waits on completions without polling Kafka, as it already does when the cap is full; change 25 removes that wait. librdkafka keeps a partition's pause across a revoke and a reassignment, so the consumer resumes every partition it is assigned: a partition that arrives starts unpaused, with a fresh ledger generation and zero charge.+- Keep polling while some partitions are paused. A paused partition still serves rebalance callbacks and keeps `max.poll.interval.ms` alive; an unpolled consumer does neither. While every partition is paused, the loop waits on completions without polling Kafka, as it already does when the cap is full; the continuous polling strategy (change 26) removes that wait. librdkafka keeps a partition's pause across a revoke and a reassignment, so the consumer resumes every partition it is assigned: a partition that arrives starts unpaused, with a fresh ledger generation and zero charge.- Watch the paused-partitions gauge and `ingestion_lag_ms`. Rollback is unsetting the values.- The budget is a fixed amount per partition, not a share of a pod-wide amount. A rebalance changes how many partitions a pod holds, not how much each one may expose, so the value does not move at rebalances.@@ -656,35 +659,79 @@ The budget is per partition, and so is the pause. One budget for the whole assig- Add `ingestion_consumer_budget_paused_partitions` (gauge) and `ingestion_consumer_budget_pauses_total{topic,partition}` (counter).-### 25. Poll continuously and remove the in-flight admission cap (structure)+### 25. Put polling behind a strategy (prepare)-**Task:** After the budget is stable on all lanes, replace the poll-and-wait loop with one loop that polls Kafka continuously and applies completions as they arrive. Remove `max_in_flight_batches`.+**Task:** Move the decisions of the poll-and-wait loop behind a polling strategy. The existing admission cap becomes the capped strategy.-**Goal:** The budget is the only admission bound. No wait on one poll holds back fetching, completions, or commits.+**Goal:** A second strategy can replace the loop by config, and the capped strategy stays as its rollback.-- The loop and the cap change together. The loop applies completions only while it waits for the oldest poll, and under steady traffic a full cap is what makes it wait. Without the cap and with the old loop, completions stay unread until every partition pauses at its budget.-- One `select!` serves the Kafka poll, the completions, the batcher's errors, the commit tick, and shutdown. Collection must be safe to cancel: the loop keeps the poll it is building in its own state and adds each message when `recv()` returns it, so a branch that wins the race drops no message.-- The commit tick fires when the pacer is next due, so a refund and the resume it allows go out on time.-- Kafka is polled while every partition is paused, so rebalance callbacks and `max.poll.interval.ms` no longer depend on completions.-- Only admission control goes. The collection bounds — batch size and batch timeout — stay: they shape one poll, not the amount of admitted work.-- Nothing tracks a poll after it is submitted. The budget bounds the admitted work, so the loop does not count polls or wait for one to finish.+- The loop and the cap change together. The old loop applies completions only while it waits for the oldest poll, and under steady traffic a full cap is what makes it wait. Without the cap and with the old loop, completions stay unread until every partition pauses at its budget. So the loop, not only the cap, changes.+- The consumer owns the only `recv()` and one `select!` over Kafka, the completions, the batcher's errors, the commit tick, shutdown, and the poll's batch timeout. It hands each event to the strategy, and the strategy returns steps: charge and submit a poll, submit a commit and resume its partitions, stop, or fail. The strategy is synchronous, so its tests feed it events without Kafka.+- Collection builds a poll message by message in a collector that the strategy keeps between events, so a branch that wins the `select!` drops no message. `StreamConsumer::recv` is cancellation safe.+- The rebalance callbacks run inside `recv()`, on the loop's task: rdkafka serves the rebalance event in the same poll that serves messages, and librdkafka delivers no fetched message of a revoked assignment after the revoke. After every wake the loop hands the revoked partitions to the strategy, before the event that woke it, also when the `select!` dropped the `recv()` that ran the callback. The strategy removes their messages from the poll in progress and from its in-flight state.+- A poll therefore holds messages of one assignment per partition, and nothing reads the ledger generation while a poll is built. The poll is charged under the current generation at submit; no `recv()` runs between the end of a poll and its charge.+- The capped strategy admits up to `max_in_flight_polls` polls and frees a slot when the oldest poll is covered. It reads Kafka and applies completions whenever the cap has room, also while every partition is paused.**Interfaces:**-- Modify `IngestionConsumer::run`: one select loop over collection, completions, errors, the commit tick, and shutdown. Remove `complete_oldest_poll`, the in-flight poll queue, and the per-poll aggregation.-- Remove `max_in_flight_batches` from `Config` and the admission bookkeeping from `IngestionConsumer`.-- Remove `ingestion_consumer_in_flight_batches`. The budget gauges replace it.+- Add the `PollingStrategy` trait: event handlers (`received`, `completed`, `revoked`, `ticked`, `shutdown`), `reading()`, `deadline()`, and `next_step() -> Option<Step>`. The consumer picks one implementation at construction and runs a loop generic over it, as it picks one `Settlement`. Add `CappedPolling`: the in-flight poll queue, the per-poll aggregation, and the admission cap.+- Add `Collector`: one poll's messages and per-partition deliveries, built one message at a time, with the batch bounds and timeout.+- Remove the per-message generation stamp from poll collection, and `FetchingPartitions`.+- Modify `IngestionConsumerOptions`: `polling` replaces `max_in_flight_batches`.++### 26. Build the continuous polling strategy (implement)++**Task:** Add a strategy that polls Kafka continuously and applies completions as they arrive. Config selects it per lane. The capped strategy stays the default.++**Goal:** With the continuous strategy, no wait on one poll holds back fetching, completions, or commits. The budget bounds each partition's replay exposure, and a pod-wide in-flight limit bounds memory.++- A poll is submitted when it reaches the batch size or the byte bound, or when the batch timeout passes after its first message. The collection bounds shape one poll, not the amount of admitted work.+- The commit tick fires at the commit interval, capped at 1s, so a refund and the resume it allows go out within one interval.+- Nothing waits on a poll after it is submitted. The consumer passes each partition's ledger generation to `Batcher::submit`, the batcher stamps it on the messages, and each `GroupCompletion` carries it back. The strategy settles a completion under that generation, so no per-poll state maps a completion to its assignment. The ledger rejects a completion of an older generation as stale. Shutdown commits everything settled and stops when no partition holds an uncommitted charge.+- `latest_processed_timestamp_ms` is set when a commit passes a poll's last offset, and the debug UI's `PollCompleted` is not emitted under the continuous strategy.+- The in-flight limit bounds the whole pod: the sum of every partition's uncommitted charge, as the budget counts it. It only gates reading, so the poll in progress still completes and the charge can pass the limit by one poll, which the batch size and the byte bound bound. The budget alone does not bound the pod, because the pod's total grows with the number of partitions it holds. The settlement returns a partition's uncommitted charge when it charges a poll and when a commit hands out offsets, and the poller keeps the latest value per partition. A poll is only the unit fed to the batcher: completions do not change the limit, and a commit frees room when it frees the budget.+- At the in-flight limit, the loop stops reading from Kafka until a commit frees room. While it waits, rebalance callbacks also wait, as they do under the capped strategy. The limit is a safety valve: size it above the budget times the partitions a pod holds, so that it does not bind in steady state.+- The continuous strategy needs `group` granularity, a budget, and an in-flight limit, each of the last two on both events and bytes: a message with no key, value or headers charges zero bytes, so a byte-only limit does not bound how many messages it admits. Without all three the consumer logs a warning and runs the capped strategy, so a lane rolled back to `poll` does not need its polling setting changed too.++**Interfaces:**++- Add `ContinuousPolling`.+- Modify `Config`: add the polling strategy (`capped` by default) and the in-flight limit in events and in bytes. Unset means no limit on that axis.++**Metrics:**++- Add `ingestion_consumer_in_flight_limit_reached_total` (counter, one per time the loop stops reading at the limit). The `kafka_consumer_ledger_uncommitted_*` gauges show each partition's share.++### 27. Poll continuously (switchover)++**Task:** After the budget is stable on all lanes, select the continuous strategy lane by lane, with an in-flight limit.++**Goal:** The budget and the in-flight limit are the only admission bounds on every lane.++- A charts values PR sets the strategy and the limit per lane. Rollback is selecting `capped`.+- Watch `ingestion_lag_ms`, the paused-partitions gauge, the in-flight gauges, and the limit counter. A limit counter that rises in steady state means the limit is too low.++### 28. Delete the capped polling strategy (cleanup)++**Task:** Remove the capped strategy after the continuous strategy is stable on all lanes.++**Goal:** One loop, and no poll-count admission cap.++**Interfaces:**++- Remove `CappedPolling`, the polling strategy setting, and `max_in_flight_batches` from `Config`.+- Remove `ingestion_consumer_in_flight_batches`. The budget gauges and the in-flight gauges replace it.## Cycle 10: partitions hand off cleanlyOutcome: a revocation drains in bounded time, and a wedged partition fails the process instead of hanging. Needs cycle 3.Until this cycle, revocation keeps today's behavior: no drain, uncommitted work replays on the new owner, and the old owner's in-flight sends finish as duplicate work. Nothing in cycles 1 to 9 changes that.-**Verify:** rebalance tests with the drain on, then the canary rebalance in change 28.+**Verify:** rebalance tests with the drain on, then the canary rebalance in change 31.rdkafka's `MockCluster` runs cooperative rebalances with no broker container, so the drain ordering is testable in the crate. The mock coordinator delays the first join by 3 s and every later round by `session.timeout.ms` minus one second, so test consumers need a session timeout above 3 s.Exit criterion: `ingestion_consumer_drain_duration_seconds` stays inside the deadline, and `ingestion_consumer_drain_dropped_messages_total` matches the expected replay volume.-### 26. Add revoke and drained markers (prepare)+### 29. Add revoke and drained markers (prepare)**Task:** Extend the batcher boundary with a revoke marker in and a drained marker out. Nothing consumes them yet.@@ -694,7 +741,7 @@ Exit criterion: `ingestion_consumer_drain_duration_seconds` stays inside the dea- Modify `Batcher`: add the marker types to the submit and completion channels. `GroupCompletion` does not change shape.-### 27. Build the drain (implement)+### 30. Build the drain (implement)**Task:** Implement the drain sequence from section 4 of the design doc, behind a config switch that ships off.@@ -725,7 +772,7 @@ Exit criterion: `ingestion_consumer_drain_duration_seconds` stays inside the dea- Add `ingestion_consumer_drain_duration_seconds` (histogram) and `ingestion_consumer_drain_dropped_messages_total` (counter): drain time and replay volume per revocation.- Add `ingestion_consumer_partition_stalls_total` (counter): stall-deadline expiries.-### 28. Enable the drain (switchover)+### 31. Enable the drain (switchover)**Task:** Turn the drain on. Canary one lane through a rebalance.@@ -737,7 +784,7 @@ Exit criterion: `ingestion_consumer_drain_duration_seconds` stays inside the dea- Modify `Config`: enable the drain.-### 29. Delete the no-drain path (cleanup)+### 32. Delete the no-drain path (cleanup)**Task:** Remove the no-drain revoke path after the drain is stable on all lanes.diff --git a/rust/ingestion-consumer/src/batcher/driver.rs b/rust/ingestion-consumer/src/batcher/driver.rsindex 98ce4d4b2d87..88b775bab316 100644--- a/rust/ingestion-consumer/src/batcher/driver.rs+++ b/rust/ingestion-consumer/src/batcher/driver.rs@@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};-use common_kafka_consumer::{AssignmentEpoch, GroupCompletion};+use common_kafka_consumer::{AssignmentEpoch, GroupCompletion, Partition};use futures::future::{BoxFuture, FutureExt};use futures::stream::{FuturesUnordered, StreamExt};use metrics::{counter, histogram};@@ -28,8 +28,7 @@ use crate::transport::{SendError, SendOutcome};use crate::types::{Accumulator, SerializedKafkaMessage};use crate::worker_registry::WorkerId;-/// A task that does not answer within this is wedged; the debug API then-/// falls back to the totals the task last published.+/// A task that does not answer within this is taken as wedged.const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(1);pub(super) struct StateMachineDriver {@@ -216,9 +215,13 @@ impl StateMachineDriver {SnapshotSender(self.inputs.clone())}- pub(super) fn submit(&self, accumulator: Accumulator) -> u64 {+ pub(super) fn submit(+ &self,+ accumulator: Accumulator,+ generations: &HashMap<Partition, u64>,+ ) -> u64 {let assignment_epoch = self.assignment_epoch.current();- let runs = KeyRun::from_groups(accumulator.into_groups());+ let runs = KeyRun::from_groups(accumulator.into_groups(), generations);let unkeyed: usize = runs.iter().filter(|run| run.messages.first().is_some_and(|m| m.key.is_none()))@@ -332,7 +335,6 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {)}Event::Response(request, Err(failure)) => {- // Backpressure is transient, not a worker fault.let cause = if failure.error.is_backpressure() {FailureCause::Busy} else {@@ -708,7 +710,6 @@ mod tests {}}- /// Lets the batcher task handle everything queued for it.async fn settle() {for _ in 0..8 {tokio::task::yield_now().await;diff --git a/rust/ingestion-consumer/src/batcher/in_flight.rs b/rust/ingestion-consumer/src/batcher/in_flight.rsindex e580a640dcc6..f8836955ce37 100644--- a/rust/ingestion-consumer/src/batcher/in_flight.rs+++ b/rust/ingestion-consumer/src/batcher/in_flight.rs@@ -22,6 +22,7 @@ impl RequestId {#[derive(Debug)]pub struct SentMessage {pub partition: i32,+ pub generation: u64,pub offset: i64,pub keyed: bool,}@@ -30,6 +31,7 @@ impl SentMessage {fn of(message: &SerializedKafkaMessage) -> Self {Self {partition: message.partition,+ generation: message.generation,offset: message.offset,keyed: message.key.is_some(),}diff --git a/rust/ingestion-consumer/src/batcher/key_queues.rs b/rust/ingestion-consumer/src/batcher/key_queues.rsindex e317fc7686ec..8a321995da5d 100644--- a/rust/ingestion-consumer/src/batcher/key_queues.rs+++ b/rust/ingestion-consumer/src/batcher/key_queues.rs@@ -6,7 +6,7 @@ use std::num::{NonZeroU32, NonZeroUsize};use std::sync::Arc;use std::time::Instant;-use common_kafka_consumer::Offset;+use common_kafka_consumer::{Offset, Partition};use super::request::RequestClass;use crate::types::{Group, SerializedKafkaMessage};@@ -19,7 +19,8 @@ pub struct KeyRun {impl KeyRun {/// An unkeyed group gets a synthetic key: it has no per-key order to keep.- pub fn from_groups(groups: Vec<Group>) -> Vec<KeyRun> {+ /// Each message takes its partition's generation from `generations`.+ pub fn from_groups(groups: Vec<Group>, generations: &HashMap<Partition, u64>) -> Vec<KeyRun> {let mut runs: Vec<KeyRun> = Vec::with_capacity(groups.len());let mut index_by_key: HashMap<Arc<str>, usize> = HashMap::new();for group in groups {@@ -30,7 +31,11 @@ impl KeyRun {format!(":{}:{}", group.partition, first).into()}};- let messages = group.messages.into_iter().map(|m| m.message);+ let generation = generations.get(&group.partition).copied().unwrap_or(0);+ let messages = group.messages.into_iter().map(|m| SerializedKafkaMessage {+ generation,+ ..m.message+ });match index_by_key.get(&routing_key) {Some(&index) => runs[index].messages.extend(messages),None => {@@ -591,8 +596,6 @@ impl KeyQueues {#[cfg(test)]impl KeyQueues {- /// Queues `messages` the way a failed request hands them back, so they- /// are ready as a replay. `routing_key` must have nothing queued.pub fn push_replay(&mut self,routing_key: Arc<str>,@@ -847,16 +850,26 @@ mod tests {}).collect(),};- let runs = KeyRun::from_groups(vec![- group(0, Some("a"), &[1]),- group(1, None, &[5]),- group(2, Some("a"), &[7]),- ]);+ let runs = KeyRun::from_groups(+ vec![+ group(0, Some("a"), &[1]),+ group(1, None, &[5]),+ group(2, Some("a"), &[7]),+ ],+ &HashMap::from([(Partition(0), 4), (Partition(2), 5)]),+ );let shapes: Vec<_> = runs.iter()- .map(|run| (&*run.routing_key, offsets(&run.messages)))+ .map(|run| {+ let generations: Vec<u64> = run.messages.iter().map(|m| m.generation).collect();+ (&*run.routing_key, offsets(&run.messages), generations)+ }).collect();- assert_eq!(shapes, vec![("a", vec![1, 7]), (":1:5", vec![5])]);+ assert_eq!(+ shapes,+ vec![("a", vec![1, 7], vec![4, 5]), (":1:5", vec![5], vec![0])],+ "each message takes its partition's generation"+ );}#[test]diff --git a/rust/ingestion-consumer/src/batcher/mod.rs b/rust/ingestion-consumer/src/batcher/mod.rsindex 6b8405ff7e4c..ab9660a644de 100644--- a/rust/ingestion-consumer/src/batcher/mod.rs+++ b/rust/ingestion-consumer/src/batcher/mod.rs@@ -1,17 +1,18 @@//! The worker batcher: the dispatch orchestration behind one boundary.//!//! The consumer loop submits one [`Accumulator`] per poll, in poll order, and-//! receives one [`GroupCompletion`] per group back. Everything in between is-//! an implementation detail of this module: the state machine queues each-//! key's messages, packs ready keys into requests for free worker slots,-//! places each request on a worker as it is sent, and retries failures; one-//! task drives it and performs its effects. No batch identity crosses the-//! boundary: the consumer correlates completions by partition and offset.+//! receives [`GroupCompletion`]s back, one per partition and generation of each+//! settled request. Everything in between is an implementation detail of this+//! module: the state machine queues each key's messages, packs ready keys into+//! requests for free worker slots, places each request on a worker as it is+//! sent, and retries failures; one task drives it and performs its effects. No+//! batch identity crosses the boundary: the consumer correlates completions by+//! partition and offset.//!//! Fatal orchestration failures (pending work that makes no progress for the-//! stall timeout, a worker that accepts more than it was sent) are reported-//! on an error channel; the consumer turns them into a process failure, so-//! the failure decision stays in the consumer loop.+//! stall timeout, a worker whose accepted count differs from what it was sent)+//! are reported on an error channel; the consumer turns them into a process+//! failure, so the failure decision stays in the consumer loop.mod driver;pub mod governor;@@ -26,10 +27,11 @@ pub(crate) mod test_support;pub mod worker_assigner;pub mod worker_pool;+use std::collections::HashMap;use std::sync::atomic::Ordering;use std::sync::Arc;-use common_kafka_consumer::GroupCompletion;+use common_kafka_consumer::{GroupCompletion, Partition};use tokio::sync::mpsc;use self::driver::StateMachineDriver;@@ -44,16 +46,12 @@ use crate::worker_registry::WorkerId;/// The batcher's output channels, held by the consumer loop.pub struct BatcherOutputs {- /// One event per group: its partition, assignment epoch, offsets, and- /// accepted count.+ /// One completion per partition and generation of each settled request.pub completions: mpsc::UnboundedReceiver<GroupCompletion>,- /// Fatal orchestration failures. The consumer fails the process on the- /// first message.+ /// Fatal orchestration failures.pub errors: mpsc::UnboundedReceiver<String>,}-/// Submits polls to the batcher state machine, and receives completions and-/// fatal errors back.pub struct Batcher {driver: StateMachineDriver,}@@ -81,13 +79,14 @@ impl Batcher {}/// Submit one poll's demuxed groups. Call on the consumer loop, in poll- /// order. Returns the assignment epoch stamped on the poll's completions.- pub fn submit(&self, accumulator: Accumulator) -> u64 {- self.driver.submit(accumulator)+ /// order. Each partition's completions carry its generation from+ /// `generations`. Returns the assignment epoch stamped on the poll's+ /// completions.+ pub fn submit(&self, accumulator: Accumulator, generations: &HashMap<Partition, u64>) -> u64 {+ self.driver.submit(accumulator, generations)}- /// A read-only view of the batcher's load, for the drain reaper, the- /// debug API and tests.+ /// A read-only view of the batcher's load.pub fn observer(&self) -> BatcherObserver {BatcherObserver {load: self.driver.load(),@@ -111,8 +110,7 @@ pub struct BatcherObserver {}impl BatcherObserver {- /// Whether the worker has sent, unresolved work. The reaper completes a- /// draining worker's drain only when it has none.+ /// Whether the worker has sent, unresolved work.pub fn has_in_flight(&self, worker: &WorkerId) -> bool {self.load.busy_workers.lock().unwrap().contains(worker)}@@ -144,9 +142,8 @@ impl BatcherObserver {}}-/// Purges revoked partitions from the batcher. Runs on the rebalance-/// callback. Purged messages never complete, so the consumer drops the-/// in-flight polls that hold a revoked partition.+/// Purges revoked partitions from the batcher. Purged messages never+/// complete.pub struct Revoker(driver::RevokeSender);impl Revoker {diff --git a/rust/ingestion-consumer/src/batcher/state_machine.rs b/rust/ingestion-consumer/src/batcher/state_machine.rsindex 073f320a50dd..6acf2fc5e58f 100644--- a/rust/ingestion-consumer/src/batcher/state_machine.rs+++ b/rust/ingestion-consumer/src/batcher/state_machine.rs@@ -244,8 +244,8 @@ impl BatcherStateMachine {}/// The new owner replays the revoked partitions, so their pending- /// messages drop. Runs already in flight finish; if they fail, their- /// messages of revoked partitions drop too.+ /// messages drop. Runs already in flight finish; their messages of revoked+ /// partitions that come back unfinished or failed drop too.pub fn on_partitions_revoked(self,now: Instant,@@ -758,23 +758,24 @@ fn completions<'a>(assignment_epoch: u64,messages: impl IntoIterator<Item = &'a SentMessage>,) -> Vec<GroupCompletion> {- let mut by_partition: HashMap<i32, Vec<Offset>> = HashMap::new();+ let mut by_partition: HashMap<(i32, u64), Vec<Offset>> = HashMap::new();for message in messages {by_partition- .entry(message.partition)+ .entry((message.partition, message.generation)).or_default().push(Offset(message.offset));}let mut completions: Vec<GroupCompletion> = by_partition.into_iter()- .map(|(partition, offsets)| GroupCompletion {+ .map(|((partition, generation), offsets)| GroupCompletion {partition: Partition(partition),assignment_epoch,+ generation,accepted: offsets.len() as u32,offsets,}).collect();- completions.sort_by_key(|completion| completion.partition.0);+ completions.sort_by_key(|completion| (completion.partition.0, completion.generation));completions}@@ -831,7 +832,6 @@ mod tests {const LATENCY_BUDGET: Duration = Duration::from_secs(1);const SLOW_RTT: Duration = Duration::from_secs(2);- /// Distinct delays, so a wakeup time shows which retry reason applied.fn retry_policy() -> RetryPolicy {RetryPolicy::new(FAULT_DELAY, BUSY_DELAY, NO_WORKER_DELAY).expect("valid retry policy")}@@ -908,8 +908,6 @@ mod tests {.collect()}- /// Applies the effects in the order the driver does, so the sentinel judges- /// the sends against the ACK marks of the same action.fn apply_to_sentinel(sentinel: &KeyOrderSentinel, effects: &Effects) -> Vec<KeyOrderViolation> {let mut batch = sentinel.batch();for ack in &effects.key_acks {@@ -1218,8 +1216,6 @@ mod tests {let (_, effects) = batcher.on_request_succeeded(now, &workers, request, 1);assert_eq!(shape(&effects.sends[0]), vec![("b", vec![2])]);- // The freed slot is refilled in the same action, so the worker is- // reported idle and then busy again.assert_eq!(effects.idle_workers, vec![WorkerId::from("w")]);assert_eq!(effects.busy_workers, vec![WorkerId::from("w")]);}@@ -1690,8 +1686,7 @@ mod tests {assert!(matches!(batcher, BatcherStateMachine::Running(_)));assert_eq!(shape(&effects.sends[0]), vec![("a", vec![1])]);- // A failure does not move the stall clock, so a replay that waited- // the budget again would be due after the stall deadline.+ // Waiting the pack budget again would pass the stall deadline.let failed_at = arrival + budget + Duration::from_secs(1);let (batcher, _) = batcher.on_request_failed(failed_at,@@ -1917,7 +1912,11 @@ mod tests {let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));let (batcher, _) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);// The new owner of the partition replays the uncommitted offsets.- let (batcher, effects) = batcher.on_groups(now, &workers, 2, vec![run("a", &[1, 2, 3])]);+ let mut replay = run("a", &[1, 2, 3]);+ for message in &mut replay.messages {+ message.generation = 2;+ }+ let (batcher, effects) = batcher.on_groups(now, &workers, 2, vec![replay]);assert!(effects.sends.is_empty(), "a's epoch-1 run is still out");let (batcher, effects) =@@ -1931,6 +1930,7 @@ mod tests {let (batcher, effects) = batcher.on_request_succeeded(now, &workers, request, 3);assert_eq!(effects.completions.len(), 1);assert_eq!(effects.completions[0].assignment_epoch, 2);+ assert_eq!(effects.completions[0].generation, 2);assert_eq!(effects.completions[0].offsets,vec![Offset(1), Offset(2), Offset(3)]@@ -1953,9 +1953,6 @@ mod tests {assert!(effects.completions.is_empty());}- /// Key `a` holds offsets 1 to 3 and key `b` offset 4. The ack for `a`- /// stops below its lowest unfinished offset, and a key with nothing- /// finished gets no ack.#[rstest]#[case::a_middle_message_of_a_shared_key(&[2], &[1, 3, 4], Some(1))]#[case::the_tail_of_a_key(&[2, 3], &[1, 4], Some(1))]diff --git a/rust/ingestion-consumer/src/batcher/test_support.rs b/rust/ingestion-consumer/src/batcher/test_support.rsindex be34eba6de51..fac51f3a16fb 100644--- a/rust/ingestion-consumer/src/batcher/test_support.rs+++ b/rust/ingestion-consumer/src/batcher/test_support.rs@@ -11,6 +11,7 @@ pub fn message(key: &str, partition: i32, offset: i64) -> SerializedKafkaMessagekey: Some(key.to_string()),value: Some("{}".to_string()),headers: HashMap::new(),+ generation: 0,}}diff --git a/rust/ingestion-consumer/src/charge_limit.rs b/rust/ingestion-consumer/src/charge_limit.rsnew file mode 100644index 000000000000..485f91963f36--- /dev/null+++ b/rust/ingestion-consumer/src/charge_limit.rs@@ -0,0 +1,43 @@+use std::num::NonZeroU64;++use common_kafka_consumer::Charge;++#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct ChargeLimit {+ events: Option<u64>,+ bytes: Option<u64>,+}++impl ChargeLimit {+ pub fn unset() -> Self {+ Self::default()+ }++ pub fn new(events: Option<NonZeroU64>, bytes: Option<NonZeroU64>) -> Self {+ Self {+ events: events.map(NonZeroU64::get),+ bytes: bytes.map(NonZeroU64::get),+ }+ }++ pub fn is_set(&self) -> bool {+ self.events.is_some() || self.bytes.is_some()+ }++ pub fn sets_both_axes(&self) -> bool {+ self.events.is_some() && self.bytes.is_some()+ }++ pub(crate) fn reached(&self, charge: Charge) -> bool {+ self.events.is_some_and(|limit| charge.events >= limit)+ || self.bytes.is_some_and(|limit| charge.bytes >= limit)+ }++ pub(crate) fn scaled(&self, fraction: f64) -> Self {+ let scale = |limit: u64| ((limit as f64 * fraction) as u64).clamp(1, limit);+ Self {+ events: self.events.map(scale),+ bytes: self.bytes.map(scale),+ }+ }+}diff --git a/rust/ingestion-consumer/src/config.rs b/rust/ingestion-consumer/src/config.rsindex de6d02bb5092..d1d11b9667e8 100644--- a/rust/ingestion-consumer/src/config.rs+++ b/rust/ingestion-consumer/src/config.rs@@ -13,6 +13,7 @@ use crate::batcher::packer::{PackTargets, Packer};use crate::batcher::retry_policy::RetryPolicy;use crate::batcher::state_machine::{BatcherStateMachine, RequestBudget};use crate::batcher::worker_assigner::WorkerAssigner;+use crate::charge_limit::ChargeLimit;use crate::discovery::DiscoveryMode;use crate::partition_budget::BudgetLimits;use crate::routing::{Router, RoutingStrategy};@@ -39,22 +40,35 @@ impl FromStr for CompletionGranularity {}}-/// Configuration for the ingestion consumer.-///-/// Kafka env vars match the Node.js ingestion consumer so this can be a-/// drop-in replacement using the same Kubernetes ConfigMap/env config.-/// The Node.js consumer reads `KAFKA_CONSUMER_*` env vars and maps them to-/// rdkafka config keys. We use the same env var names and defaults here.-///-/// Any `KAFKA_CONSUMER_*` env var is also read and applied as an rdkafka-/// config override — e.g. `KAFKA_CONSUMER_METADATA_BROKER_LIST=kafka:9092`-/// becomes `metadata.broker.list=kafka:9092`. This matches the Node.js-/// `getKafkaConfigFromEnv('CONSUMER')` behavior.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]+pub enum Polling {+ #[default]+ Capped,+ Continuous,+}++impl FromStr for Polling {+ type Err = String;++ fn from_str(value: &str) -> Result<Self, Self::Err> {+ match value.trim().to_lowercase().as_str() {+ "capped" => Ok(Self::Capped),+ "continuous" => Ok(Self::Continuous),+ other => Err(format!(+ "unknown consumer polling strategy '{other}' (expected 'capped' or 'continuous')"+ )),+ }+ }+}++/// Configuration for the ingestion consumer. Env var names and defaults match+/// the Node.js ingestion consumer, so both read the same ConfigMap.+/// `KAFKA_CONSUMER_*` vars also apply as rdkafka overrides; see+/// `parse_kafka_consumer_env_overrides`.#[derive(Envconfig, Clone, Debug)]pub struct Config {// ---- Kafka connection (matches Node.js KafkaConsumer defaults) ----- /// Kafka broker list. Overridable via KAFKA_CONSUMER_METADATA_BROKER_LIST- /// (same as Node.js). This is used as the base default.+ /// Kafka broker list. `KAFKA_CONSUMER_METADATA_BROKER_LIST` overrides it.#[envconfig(default = "kafka:9092")]pub kafka_hosts: String,@@ -91,7 +105,6 @@ pub struct Config {#[envconfig(default = "300000")]pub kafka_consumer_max_poll_interval_ms: u32,- /// 1MB — matches Node.js default#[envconfig(default = "1048576")]pub kafka_consumer_max_partition_fetch_bytes: u32,@@ -113,18 +126,16 @@ pub struct Config {#[envconfig(default = "52428800")]pub kafka_consumer_fetch_max_bytes: u32,- /// 10MB — matches Node.js fetch.message.max.bytes#[envconfig(default = "10485760")]pub kafka_consumer_fetch_message_max_bytes: u32,- /// 50ms — matches Node.js default (aggressive fetch for low latency)+ /// Below the librdkafka default of 500ms, for low fetch latency.#[envconfig(default = "50")]pub kafka_consumer_fetch_wait_max_ms: u32,#[envconfig(default = "100000")]pub kafka_consumer_queued_min_messages: u32,- /// 100MB — matches Node.js default (reduced from rdkafka default of 1GB)#[envconfig(default = "102400")]pub kafka_consumer_queued_max_messages_kbytes: u32,@@ -137,8 +148,8 @@ pub struct Config {#[envconfig(default = "15000")]pub kafka_consumer_statistics_interval_ms: u32,- /// Pod hostname from K8s, used as client.id and group.instance.id- /// for sticky partition assignment (same as Node.js hostname())+ /// Pod hostname from K8s. Used as `client.id`, and as `group.instance.id`+ /// when `KAFKA_CONSUMER_STATIC_MEMBERSHIP` is on.#[envconfig(from = "HOSTNAME")]pub pod_hostname: Option<String>,@@ -159,26 +170,10 @@ pub struct Config {#[envconfig(default = "500")]pub consumer_batch_size: usize,- /// Maximum payload bytes to collect before dispatching a batch, in KiB.- /// `0` (default) disables the bound, leaving collection count-only.- ///- /// Resident memory is `CONSUMER_MAX_BACKGROUND_TASKS x batch x event size`,- /// doubled while a sub-batch is in flight, so a count-only cap only bounds- /// memory if every lane's event size is known and stable. It is not: measured- /// mean payloads across analytics lanes span ~1KB to ~43KB, a 45x spread, and- /// the same `CONSUMER_BATCH_SIZE` therefore means 45x the memory on one lane- /// versus another. This bound restates the limit in the unit that actually- /// constrains the pod, so one value is safe everywhere.- ///- /// Whichever bound is reached first ends collection. The count cap stays- /// useful as a downstream-shape guard (worker request size, per-batch- /// overhead); this one is the memory guard. Checked before appending, so a- /// batch always carries at least one message and overshoot is bounded by a- /// single message (itself capped by `fetch.message.max.bytes`).- ///- /// Default is off so that rolling this image out changes no lane's batch- /// composition; lanes opt in where their memory arithmetic is already- /// written down.+ /// Maximum payload bytes to collect into one poll, in KiB. `0` (default)+ /// disables the bound. Whichever of this and `CONSUMER_BATCH_SIZE` is reached+ /// first ends collection: the count bounds request shape, this bounds memory,+ /// because event size differs per lane. A poll can pass it by one message.#[envconfig(from = "CONSUMER_BATCH_SIZE_KB", default = "0")]pub consumer_batch_size_kb: usize,@@ -265,8 +260,8 @@ pub struct Config {#[envconfig(from = "INGESTION_GOVERNOR_STEP_INTERVAL_MS", default = "1000")]pub governor_step_interval_ms: u64,- /// Maximum Kafka batches to process concurrently. Matches the Node.js- /// CONSUMER_MAX_BACKGROUND_TASKS setting used by the Kafka consumer wrapper.+ /// Maximum polls in flight under the capped polling strategy; continuous+ /// polling does not read it. Same name as the Node.js setting.#[envconfig(from = "CONSUMER_MAX_BACKGROUND_TASKS", default = "1")]pub consumer_max_background_tasks: usize,@@ -295,6 +290,26 @@ pub struct Config {#[envconfig(from = "CONSUMER_PARTITION_BUDGET_LOW_WATERMARK", default = "0.8")]pub consumer_partition_budget_low_watermark: f64,+ /// `capped` or `continuous`. `continuous` runs only with `group`+ /// granularity, a partition budget, and an in-flight limit; without all+ /// three the consumer runs `capped`.+ #[envconfig(from = "CONSUMER_POLLING_STRATEGY", default = "capped")]+ pub consumer_polling_strategy: Polling,++ /// Uncommitted events across all partitions at which the continuous+ /// strategy stops reading from Kafka. The poll in progress still+ /// completes, so the total can pass it by `CONSUMER_BATCH_SIZE`. `0`+ /// (default) sets no event limit.+ #[envconfig(from = "CONSUMER_IN_FLIGHT_EVENTS", default = "0")]+ pub consumer_in_flight_events: u64,++ /// Uncommitted key-plus-value-plus-header bytes across all partitions at+ /// which the continuous strategy stops reading from Kafka. The poll in+ /// progress still completes, so the total can pass it by one poll. `0`+ /// (default) sets no byte limit.+ #[envconfig(from = "CONSUMER_IN_FLIGHT_BYTES", default = "0")]+ pub consumer_in_flight_bytes: u64,+// ---- Debug API ----/// Serve the real-time debug API (`/debug/load`, `/debug/state`,/// `/debug/events` SSE) on the health server, recording structured events@@ -337,7 +352,7 @@ pub struct Config {/// Cap on the estimated serialized size of one sub-batch frame (bytes)./// Sub-batches above it ride the worker stream as consecutive chunks. Must/// stay under the worker's gRPC message limit with headroom for the size- /// estimate. Default 10 MiB.+ /// estimate.#[envconfig(from = "INGESTION_TRANSPORT_MAX_BODY_BYTES", default = "10485760")]pub transport_max_body_bytes: usize,@@ -353,12 +368,10 @@ pub struct Config {#[envconfig(from = "INGESTION_WORKER_GRPC_PORT_OFFSET", default = "0")]pub ingestion_worker_grpc_port_offset: u16,- /// Fence a worker stream (fail its un-acked and queued sub-batches into the- /// deferral path, reconnect, re-route) when un-acked work sees no ack for- /// this long (milliseconds). The stream has no per-send timeout, so this- /// watchdog is what turns a worker that stops acking into a re-route- /// instead of a silent forever-wait. Sized above the worst-case batch- /// processing time, like the HTTP timeout it replaces.+ /// Fence a worker stream (fail its un-acked and queued sub-batches back to+ /// the caller, reconnect) when one sub-batch waits this long for its ack+ /// (milliseconds). The stream has no other per-send timeout. Size it above+ /// the worst-case batch processing time.#[envconfig(from = "INGESTION_WORKER_STREAM_ACK_TIMEOUT_MS", default = "60000")]pub ingestion_worker_stream_ack_timeout_ms: u64,@@ -400,9 +413,7 @@ pub struct Config {/// Minimum aperture width for `INGESTION_ROUTING_STRATEGY=aperture`: how/// many workers this dispatcher's ring slice spans. The effective width/// is floored at `ceil(pool / dispatchers)` so the fleet's slices always- /// cover the whole pool, no matter the pool/dispatcher ratio. Small- /// values maximize sub-batch consolidation; the width becomes- /// feedback-driven in a later stage.+ /// cover the whole pool. Small values maximize sub-batch consolidation.#[envconfig(from = "INGESTION_MIN_APERTURE", default = "3")]pub min_aperture: usize,@@ -610,6 +621,13 @@ impl Config {)}+ pub fn in_flight_limit(&self) -> ChargeLimit {+ ChargeLimit::new(+ NonZeroU64::new(self.consumer_in_flight_events),+ NonZeroU64::new(self.consumer_in_flight_bytes),+ )+ }+const MIN_REQUEST_SOFT_BUDGET_MS: u64 = 100;pub fn request_budget(&self) -> Result<Option<RequestBudget>, String> {@@ -671,13 +689,10 @@ impl Config {.collect()}- /// Build a fully-configured rdkafka ClientConfig.- ///- /// Mirrors the Node.js KafkaConsumer constructor config with the same- /// defaults and override priority:- /// 1. Hardcoded defaults (from struct field defaults)- /// 2. `KAFKA_CONSUMER_*` env var overrides (same as Node.js getKafkaConfigFromEnv)- /// 3. Non-overridable settings (partition.assignment.strategy, enable.auto.offset.store)+ /// Build a fully-configured rdkafka ClientConfig, in Node.js order:+ /// 1. Defaults from the struct fields.+ /// 2. `KAFKA_CONSUMER_*` env var overrides.+ /// 3. Classic-protocol keys removed when `group.protocol=consumer`.pub fn build_consumer_config(&self) -> ClientConfig {let mut builder = ConsumerConfigBuilder::for_batch_consumer(&self.kafka_hosts,@@ -724,7 +739,6 @@ impl Config {builder = builder.set("client.rack", &self.kafka_client_rack);}- // Apply KAFKA_CONSUMER_* env var overrides (same as Node.js getKafkaConfigFromEnv)let overrides = parse_kafka_consumer_env_overrides();for (key, value) in &overrides {info!(key = %key, value = %value, "Applying KAFKA_CONSUMER_ env override");diff --git a/rust/ingestion-consumer/src/consumer.rs b/rust/ingestion-consumer/src/consumer.rsindex 07b77244e19c..e58084713362 100644--- a/rust/ingestion-consumer/src/consumer.rs+++ b/rust/ingestion-consumer/src/consumer.rs@@ -1,309 +1,43 @@-use std::collections::{HashMap, HashSet, VecDeque};-use std::sync::atomic::{AtomicBool, Ordering};+use std::collections::HashMap;use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};use common_kafka_consumer::{- AssignmentEpoch, Charge, GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,+ AssignmentEpoch, GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,};-use futures::StreamExt;use lifecycle::Handle;use metrics::{counter, gauge, histogram};use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};-use rdkafka::message::{Headers, Message};+use rdkafka::error::KafkaError;+use rdkafka::message::BorrowedMessage;use rdkafka::TopicPartitionList;use tokio::sync::mpsc;+use tokio::time::MissedTickBehavior;use tracing::{error, info, warn};use crate::batcher::{make_batch_id, Batcher, BatcherObserver, BatcherOutputs, Revoker};-use crate::config::{CompletionGranularity, Config};-use crate::debug_recorder::{- record_if, CommitOffset, DebugEventKind, DebugRecorder, PartitionOffset, PartitionOffsets,-};+use crate::charge_limit::ChargeLimit;+use crate::config::{CompletionGranularity, Config, Polling};+use crate::debug_recorder::{record_if, CommitOffset, DebugEventKind, DebugRecorder};use crate::discovery::DiscoveryMode;use crate::grpc_transport::GrpcTransport;use crate::ledger_rejection::{report_rejection, RejectedSlice};-use crate::order_sentinel::OffsetSpan;use crate::partition_budget::BudgetLimits;+use crate::polling::{+ self, debug_partition_offsets, debug_partition_spans, BatchStats, CappedPolling,+ CollectedBatch, ContinuousPolling, InFlightPoll, PartitionDeliveries, Parts, PollLimits,+ PollingStrategy, Selected, Step,+};use crate::rebalance::{partition_error, PartitionHook, RebalanceContext};-use crate::settlement::{self, ChargedSlice, Commit, Pause, Settlement};-use crate::types::{Accumulator, SerializedKafkaMessage};--/// Batch-wide statistics gathered while collecting, used to emit parity-/// metrics. Per-partition facts live on [`PartitionDeliveries`].-struct BatchStats {- /// Per-message (partition, lag_ms) pairs — for `ingestion_lag_ms_histogram`.- message_lags_ms: Vec<(i32, i64)>,- /// Total byte size of message payloads — for `consumer_batch_size_kb`.- total_bytes: usize,-}--impl BatchStats {- fn new() -> Self {- Self {- message_lags_ms: Vec::new(),- total_bytes: 0,- }- }-}--/// One delivered message, as the batch records it against its partition.-struct Delivery {- offset: i64,- charge: Charge,- /// Kafka message timestamp (ms).- kafka_ts: i64,- /// Ingestion lag (ms); `None` without a parseable `now` header.- lag_ms: Option<i64>,-}--/// What a batch saw delivered from one partition, folded in one map entry-/// per message. The span bounds what the batch delivered; the stamped-/// charges feed the ledger.-struct PartitionDeliveries {- /// The offsets the batch delivered, first to last.- span: OffsetSpan,- /// Ledger generation the charges are stamped with.- generation: u64,- /// Ledger generations version seen when the charges were last stamped.- /// Comparing it per message is cheaper than reading the generation.- generations_version_seen: u64,- /// The slice charged to the ledger: offsets delivered under `generation`.- charges: Vec<(Offset, Charge)>,- /// Max Kafka message timestamp (ms) — for `latest_processed_timestamp_ms`.- latest_kafka_ts: i64,- /// Max ingestion lag (ms) — for `ingestion_lag_ms`.- max_lag_ms: Option<i64>,- /// Messages the poll delivered, covered, and accepted from this- /// partition, so a revoked partition can leave the poll exactly.- delivered: u32,- covered: u32,- accepted: u32,- /// The offsets a completion covered. Keyed by offset rather than by- /// position in `span`, because a poll can span an offset gap.- credited: HashSet<i64, ahash::RandomState>,-}--impl PartitionDeliveries {- fn new(generation: u64, generations_version: u64, delivery: &Delivery) -> Self {- Self {- span: OffsetSpan::new(delivery.offset),- generation,- generations_version_seen: generations_version,- charges: vec![(Offset(delivery.offset), delivery.charge)],- latest_kafka_ts: delivery.kafka_ts,- max_lag_ms: delivery.lag_ms,- delivered: 1,- covered: 0,- accepted: 0,- credited: HashSet::default(),- }- }-- /// Credit a completion's run and return the offsets to settle: those- /// charged under this generation that no earlier completion credited.- fn credit_run(&mut self, run: &[Offset]) -> Vec<Offset> {- let Self {- credited, charges, ..- } = self;- run.iter()- .copied()- .filter(|offset| {- credited.insert(offset.0)- && charges- .binary_search_by_key(&offset.0, |(charged, _)| charged.0)- .is_ok()- })- .collect()- }-- fn uncredited_charge(&self) -> Option<Offset> {- self.charges- .iter()- .map(|(offset, _)| *offset)- .find(|offset| !self.credited.contains(&offset.0))- }-- /// Record one more delivery. `generation` is consulted only when- /// `generations_version` moved since the last stamp. A moved generation- /// means the partition was revoked and regained inside this batch: the- /// offsets buffered so far belong to the old assignment and Kafka- /// redelivers them, so the ledger slice restarts. The span keeps them,- /// so the commit sentinel still sees the whole delivered range.- fn record(- &mut self,- generations_version: u64,- generation: impl FnOnce() -> u64,- delivery: &Delivery,- ) {- self.span.extend(delivery.offset);- self.delivered += 1;- self.latest_kafka_ts = self.latest_kafka_ts.max(delivery.kafka_ts);- if let Some(lag_ms) = delivery.lag_ms {- self.max_lag_ms = Some(self.max_lag_ms.map_or(lag_ms, |max| max.max(lag_ms)));- }- if generations_version != self.generations_version_seen {- self.generations_version_seen = generations_version;- let generation = generation();- if generation != self.generation {- self.generation = generation;- self.charges.clear();- }- }- self.charges- .push((Offset(delivery.offset), delivery.charge));- }-}--/// Why the consumer loop stops. The two are reported on separate channels-/// so a dashboard can tell a pipeline failure from a client that can no-/// longer commit.-enum Failure {- /// A poll could not be collected, dispatched, or completed.- Batch(anyhow::Error),- /// An offset commit could not be submitted to the Kafka client.- Commit(anyhow::Error),-}--/// Output of `collect_batch`.-struct CollectedBatch {- /// The poll's messages, demuxed per partition and routing key.- accumulator: Accumulator,- partitions: HashMap<TopicPartition, PartitionDeliveries>,- stats: BatchStats,-}--/// One submitted poll, awaiting its group completions. The consumer-/// correlates completions to it by assignment epoch, partition, and offset;-/// the poll retires only once completions cover every message.-struct InFlightPoll {- /// Consumer-side id for logs and debug events only; the batcher's- /// internal batch id never crosses the boundary.- poll_id: String,- /// The epoch the batcher stamped on this poll's completions.- assignment_epoch: u64,- partitions: HashMap<TopicPartition, PartitionDeliveries>,- message_count: u32,- /// Messages covered by completions so far, accepted or not.- covered: u32,- /// Worker-accepted messages so far. The poll retires only when this- /// reaches `message_count`.- accepted: u32,- dispatched_at: Instant,-}--impl InFlightPoll {- fn is_complete(&self) -> bool {- self.covered >= self.message_count- }-- /// A charged offset that no completion covered. A complete poll can still- /// have one, because one offset credited twice fills `covered` as well.- fn uncredited_charge(&self) -> Option<(&TopicPartition, Offset)> {- self.partitions- .iter()- .find_map(|(topic_partition, deliveries)| {- deliveries- .uncredited_charge()- .map(|offset| (topic_partition, offset))- })- }-- fn charged_slices(&self) -> Vec<ChargedSlice<'_>> {- self.partitions- .iter()- .map(|(topic_partition, deliveries)| ChargedSlice {- topic_partition,- generation: deliveries.generation,- span: deliveries.span,- charges: &deliveries.charges,- })- .collect()- }-}--/// A partition revocation stamped with the ledger generation started by the-/// revoke. Poll slices from earlier generations are stale; slices collected-/// after a later reassignment must survive a delayed notification.-#[derive(Clone, Debug)]-struct RevokedPartition {- topic_partition: TopicPartition,- generation: u64,-}--#[derive(Default)]-struct Partitions {- fetching: HashSet<TopicPartition>,- paused: HashSet<TopicPartition>,-}--#[derive(Default)]-struct FetchingPartitions {- partitions: Mutex<Partitions>,- all_paused: AtomicBool,-}--impl FetchingPartitions {- fn assigned(&self, topic_partitions: &[TopicPartition]) {- self.update(|partitions| {- for topic_partition in topic_partitions {- partitions.paused.remove(topic_partition);- partitions.fetching.insert(topic_partition.clone());- }- });- }-- fn revoked(&self, topic_partitions: &[TopicPartition]) {- self.update(|partitions| {- for topic_partition in topic_partitions {- partitions.paused.remove(topic_partition);- partitions.fetching.remove(topic_partition);- }- });- }-- fn paused(&self, topic_partition: &TopicPartition) {- self.update(|partitions| {- if partitions.fetching.remove(topic_partition) {- partitions.paused.insert(topic_partition.clone());- }- });- }-- fn resumed(&self, topic_partition: &TopicPartition) {- self.update(|partitions| {- if partitions.paused.remove(topic_partition) {- partitions.fetching.insert(topic_partition.clone());- }- });- }-- fn all_paused(&self) -> bool {- self.all_paused.load(Ordering::Relaxed)- }-- fn update(&self, change: impl FnOnce(&mut Partitions)) {- let mut partitions = self.partitions.lock().unwrap();- change(&mut partitions);- self.all_paused.store(- !partitions.paused.is_empty() && partitions.fetching.is_empty(),- Ordering::Relaxed,- );- }-}+use crate::settlement::{self, Commit, Settlement};-/// The consumer's revocation hook: purge the batcher, and record the revoked-/// partitions, because the purge means their polls never complete.fn revoke_hook(revoker: Revoker,- ledger: Arc<TopicOffsetLedger>,settlement: Arc<dyn Settlement>,- revoked_partitions: Arc<Mutex<Vec<RevokedPartition>>>,- fetching: Arc<FetchingPartitions>,+ revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,) -> PartitionHook {Box::new(move |partitions| {settlement.forget_partitions(partitions);- fetching.revoked(partitions);let pairs: Vec<(String, i32)> = partitions.iter().map(|tp| (tp.topic.clone(), tp.partition))@@ -312,177 +46,44 @@ fn revoke_hook(revoked_partitions.lock().unwrap()- .extend(partitions.iter().map(|topic_partition| RevokedPartition {- topic_partition: topic_partition.clone(),- generation: ledger.generation(topic_partition),- }));+ .extend_from_slice(partitions);})}fn assign_hook(settlement: Arc<dyn Settlement>,- fetching: Arc<FetchingPartitions>,assignment_epoch: AssignmentEpoch,+ revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,) -> PartitionHook {Box::new(move |partitions| {// Also covers a loss with no revoke callback (an error rebalance, a fenced member).settlement.forget_partitions(partitions);- fetching.assigned(partitions);assignment_epoch.bump();+ revoked_partitions+ .lock()+ .unwrap()+ .extend_from_slice(partitions);})}fn wire_rebalance_hooks(context: &RebalanceContext,revoker: Revoker,- ledger: &Arc<TopicOffsetLedger>,settlement: &Arc<dyn Settlement>,assignment_epoch: AssignmentEpoch,-) -> (Arc<Mutex<Vec<RevokedPartition>>>, Arc<FetchingPartitions>) {+) -> Arc<Mutex<Vec<TopicPartition>>> {let revoked_partitions = Arc::new(Mutex::new(Vec::new()));- let fetching = Arc::new(FetchingPartitions::default());context.set_on_revoke(revoke_hook(revoker,- Arc::clone(ledger),Arc::clone(settlement),Arc::clone(&revoked_partitions),- Arc::clone(&fetching),));context.set_on_assign(assign_hook(Arc::clone(settlement),- Arc::clone(&fetching),assignment_epoch,+ Arc::clone(&revoked_partitions),));- (revoked_partitions, fetching)-}--/// Remove revoked partitions from in-flight polls, and remove polls left-/// empty. Only slices from before that partition's revoke generation go: a-/// poll's kept or newly reassigned slices retain their counts and settle their-/// ledger charges at commit, so the frontier never crosses a hole.-fn strip_revoked_partitions(- in_flight_polls: &mut VecDeque<InFlightPoll>,- revoked: &[RevokedPartition],-) -> u64 {- let mut stripped: u64 = 0;- for poll in in_flight_polls.iter_mut() {- let mut removed_delivered = 0u32;- let mut removed_covered = 0u32;- let mut removed_accepted = 0u32;- poll.partitions.retain(|topic_partition, deliveries| {- if revoked.iter().any(|revoked| {- revoked.topic_partition == *topic_partition- && deliveries.generation < revoked.generation- }) {- removed_delivered += deliveries.delivered;- removed_covered += deliveries.covered;- removed_accepted += deliveries.accepted;- false- } else {- true- }- });- if removed_delivered > 0 {- stripped += u64::from(removed_delivered);- poll.message_count = poll.message_count.saturating_sub(removed_delivered);- poll.covered = poll.covered.saturating_sub(removed_covered);- poll.accepted = poll.accepted.saturating_sub(removed_accepted);- }- }- in_flight_polls.retain(|poll| poll.message_count > 0);- stripped-}--/// Credit each of a completion's offsets to the poll that contains it: the-/// one collected under the same assignment epoch whose span holds the offset.-/// Within one epoch, poll spans are disjoint per partition, so at most one-/// poll matches per offset. A key-table run merges messages from several-/// polls into one send, so one completion can span polls; crediting per-/// offset keeps every poll's count exact. Acceptance credits the leading-/// offsets, so a worker that under-reports shorts the tail poll's accepted-/// check, matching [`send_group_completions`]'s split across groups. An-/// offset that matches no poll (its partition was revoked and reassigned-/// while the group was out, or its poll is gone) is discarded; a completion-/// with any discarded offset counts as stale once.-///-/// [`send_group_completions`]: crate::batcher-fn apply_completion(- in_flight: &mut VecDeque<InFlightPoll>,- completion: &GroupCompletion,- mut settle: impl FnMut(&TopicPartition, u64, &[Offset]) -> anyhow::Result<()>,-) -> anyhow::Result<()> {- let fully_accepted = completion.accepted as usize == completion.offsets.len();- let mut accepted = completion.accepted;- let mut unmatched: u64 = 0;- let mut remaining = completion.offsets.as_slice();- 'runs: while let Some(first) = remaining.first() {- for poll in in_flight- .iter_mut()- .filter(|poll| poll.assignment_epoch == completion.assignment_epoch)- {- for (topic_partition, deliveries) in poll.partitions.iter_mut() {- let span = deliveries.span;- if topic_partition.partition != completion.partition.0- || first.0 < span.first- || span.last < first.0- {- continue;- }- let run_len = remaining- .iter()- .take_while(|offset| span.first <= offset.0 && offset.0 <= span.last)- .count();- let (run, rest) = remaining.split_at(run_len);- let run_accepted = accepted.min(run_len as u32);- accepted -= run_accepted;- poll.covered += run_len as u32;- poll.accepted += run_accepted;- deliveries.covered += run_len as u32;- deliveries.accepted += run_accepted;- let settleable = deliveries.credit_run(run);- if fully_accepted && !settleable.is_empty() {- settle(topic_partition, deliveries.generation, &settleable)?;- }- remaining = rest;- continue 'runs;- }- }- unmatched += 1;- remaining = &remaining[1..];- }- if unmatched > 0 {- counter!("ingestion_consumer_stale_group_completions_total").increment(1);- warn!(- partition = %completion.partition,- unmatched,- epoch = completion.assignment_epoch,- "Discarding completion offsets that match no in-flight poll"- );- }- Ok(())-}--/// Pops the front poll for its commit. A complete poll commits only when-/// workers accepted all of it and a completion covered every charged offset.-fn pop_committable(in_flight: &mut VecDeque<InFlightPoll>) -> anyhow::Result<InFlightPoll> {- let poll = in_flight.pop_front().expect("front is present");- if poll.accepted < poll.message_count {- anyhow::bail!(- "accepted {}/{} messages — not committing offsets",- poll.accepted,- poll.message_count- );- }- if let Some((topic_partition, offset)) = poll.uncredited_charge() {- counter!("ingestion_consumer_commit_violations_total", "kind" => "uncredited").increment(1);- anyhow::bail!(- "offset {} of {}/{} has no completion — not committing offsets",- offset.0,- topic_partition.topic,- topic_partition.partition- );- }- Ok(poll)+ revoked_partitions}fn commit_tick(granularity: CompletionGranularity, commit_interval: Duration) -> Duration {@@ -498,16 +99,17 @@ fn commit_tick(granularity: CompletionGranularity, commit_interval: Duration) ->}/// Options for constructing an [`IngestionConsumer`] from pre-built parts.-/// Used in integration tests where the Kafka consumer is created externally.pub struct IngestionConsumerOptions {pub batch_size: usize,/// Payload-byte bound on a batch; `0` disables it (count-only collection)./// See `Config::consumer_batch_size_kb`.pub batch_size_bytes: usize,pub batch_timeout: Duration,+ /// The capped strategy's admission cap, in polls.pub max_in_flight_batches: usize,+ pub polling: Polling,+ pub in_flight_limit: ChargeLimit,pub group_id: String,- /// Debug event recorder; `None` unless `DEBUG_API_ENABLED`.pub debug_recorder: Option<Arc<DebugRecorder>>,pub completion_granularity: CompletionGranularity,pub commit_interval: Duration,@@ -516,8 +118,8 @@ pub struct IngestionConsumerOptions {}/// The main consumer loop: reads from Kafka, demuxes each poll into groups,-/// submits them to the [`Batcher`] (which routes, dispatches, and flushes),-/// and commits offsets once the batcher's completions cover a poll or a group.+/// submits them to the [`Batcher`], and commits offsets once the batcher's+/// completions cover a poll or a group.pub struct IngestionConsumer {consumer: Arc<StreamConsumer<RebalanceContext>>,batcher: Batcher,@@ -528,23 +130,21 @@ pub struct IngestionConsumer {batch_size: usize,batch_size_bytes: usize,batch_timeout: Duration,- max_in_flight_batches: usize,+ polling: Selected,commit_tick: Duration,handle: Handle,group_id: String,- /// Debug event recorder; `None` unless `DEBUG_API_ENABLED`.debug_recorder: Option<Arc<DebugRecorder>>,topic_offset_ledger: Arc<TopicOffsetLedger>,/// Partitions revoked since the loop last looked, fed by the rebalance/// callback.- revoked_partitions: Arc<Mutex<Vec<RevokedPartition>>>,- fetching: Arc<FetchingPartitions>,+ revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,settlement: Arc<dyn Settlement>,}impl IngestionConsumer {- /// Constructs a consumer from pre-built parts. Useful in integration tests- /// where the Kafka consumer is created and subscribed externally.+ /// Constructs a consumer from parts the caller builds. The caller must+ /// subscribe `consumer` to its topic.pub fn from_parts(consumer: StreamConsumer<RebalanceContext>,batcher: Batcher,@@ -555,16 +155,22 @@ impl IngestionConsumer {handle: Handle,) -> Self {let topic_offset_ledger = options.ledger;+ let polling = polling::select(+ options.polling,+ options.completion_granularity,+ &options.partition_budget,+ options.in_flight_limit,+ options.max_in_flight_batches,+ );let settlement = settlement::for_granularity(options.completion_granularity,Arc::clone(&topic_offset_ledger),options.commit_interval,options.partition_budget,);- let (revoked_partitions, fetching) = wire_rebalance_hooks(+ let revoked_partitions = wire_rebalance_hooks(consumer.context(),batcher.revoker(),- &topic_offset_ledger,&settlement,transport.assignment_epoch(),);@@ -573,7 +179,6 @@ impl IngestionConsumer {debug_recorder: options.debug_recorder,topic_offset_ledger,revoked_partitions,- fetching,settlement,consumer,batcher,@@ -583,7 +188,7 @@ impl IngestionConsumer {batch_size: options.batch_size,batch_size_bytes: options.batch_size_bytes,batch_timeout: options.batch_timeout,- max_in_flight_batches: options.max_in_flight_batches.max(1),+ polling,commit_tick: commit_tick(options.completion_granularity, options.commit_interval),handle,group_id: options.group_id,@@ -598,9 +203,8 @@ impl IngestionConsumer {handle: Handle,debug_recorder: Option<Arc<DebugRecorder>>,) -> anyhow::Result<Self> {- // In endpointslice mode the worker set comes from discovery, so there is- // no static readiness list — main gates startup on the first discovered- // worker. In static mode we keep the configured list for readiness.+ // Endpointslice mode discovers workers at runtime, so it has no static+ // readiness list.let worker_urls = match config.worker_discovery_mode {DiscoveryMode::Static => config.worker_urls(),DiscoveryMode::EndpointSlice => Vec::new(),@@ -621,19 +225,26 @@ impl IngestionConsumer {.key_order_sentinel().set_enabled(config.consumer_order_sentinel_enabled);let topic_offset_ledger = Arc::new(TopicOffsetLedger::new());+ let partition_budget = config+ .partition_budget()+ .map_err(|err| anyhow::anyhow!("invalid partition budget: {err}"))?;+ let polling = polling::select(+ config.consumer_polling_strategy,+ config.consumer_completion_granularity,+ &partition_budget,+ config.in_flight_limit(),+ config.consumer_max_background_tasks,+ );let settlement = settlement::for_granularity(config.consumer_completion_granularity,Arc::clone(&topic_offset_ledger),Duration::from_millis(config.consumer_commit_interval_ms),- config- .partition_budget()- .map_err(|err| anyhow::anyhow!("invalid partition budget: {err}"))?,+ partition_budget,);let context = RebalanceContext::new();- let (revoked_partitions, fetching) = wire_rebalance_hooks(+ let revoked_partitions = wire_rebalance_hooks(&context,batcher.revoker(),- &topic_offset_ledger,&settlement,transport.assignment_epoch(),);@@ -649,6 +260,7 @@ impl IngestionConsumer {batch_size = config.consumer_batch_size,batch_size_kb = config.consumer_batch_size_kb,completion_granularity = ?config.consumer_completion_granularity,+ polling = ?polling,"Kafka consumer subscribed");@@ -657,7 +269,6 @@ impl IngestionConsumer {debug_recorder,topic_offset_ledger,revoked_partitions,- fetching,settlement,batcher,outputs: Some(outputs),@@ -666,7 +277,7 @@ impl IngestionConsumer {batch_size: config.consumer_batch_size,batch_size_bytes: config.consumer_batch_size_kb.saturating_mul(1024),batch_timeout: Duration::from_millis(config.consumer_batch_timeout_ms),- max_in_flight_batches: config.consumer_max_background_tasks.max(1),+ polling,commit_tick: commit_tick(config.consumer_completion_granularity,Duration::from_millis(config.consumer_commit_interval_ms),@@ -676,13 +287,12 @@ impl IngestionConsumer {})}- /// A read-only view of the batcher's load.pub fn batcher_observer(&self) -> BatcherObserver {self.batcher.observer()}- /// Run the consumer loop until shutdown is signalled via the lifecycle handle.- /// Waits for all workers to be ready before starting to consume from Kafka.+ /// Runs the consumer loop until the lifecycle handle signals shutdown.+ /// First waits until the statically configured workers are ready.pub async fn process(mut self) {let _guard = self.handle.process_scope();let BatcherOutputs {@@ -708,88 +318,153 @@ impl IngestionConsumer {workers: self.worker_urls.clone(),});- self.run(completions, errors).await;+ let parts = Parts {+ limits: PollLimits {+ batch_size: self.batch_size,+ batch_size_bytes: self.batch_size_bytes,+ batch_timeout: self.batch_timeout,+ },+ settlement: Arc::clone(&self.settlement),+ group_id: self.group_id.clone(),+ debug_recorder: self.debug_recorder.clone(),+ };+ match self.polling {+ Selected::Capped {+ max_in_flight_polls,+ } => {+ let poller = CappedPolling::new(parts, max_in_flight_polls);+ self.run(poller, completions, errors).await;+ }+ Selected::Continuous { in_flight } => {+ let poller = ContinuousPolling::new(parts, in_flight);+ self.run(poller, completions, errors).await;+ }+ }if let Err(err) = self.commit_all() {warn!(error = %err, "Could not submit pending frontiers on exit");}info!("Consumer loop stopped");}- /// Poll, dispatch, and complete until shutdown drains the in-flight polls- /// or the loop fails.- async fn run(+ async fn run<P: PollingStrategy>(&self,+ mut poller: P,mut completions: mpsc::UnboundedReceiver<GroupCompletion>,mut errors: mpsc::UnboundedReceiver<String>,) {- let mut in_flight_polls: VecDeque<InFlightPoll> = VecDeque::new();+ let mut tick = tokio::time::interval(self.commit_tick);+ tick.set_missed_tick_behavior(MissedTickBehavior::Delay);+ let poll_deadline = tokio::time::sleep(Duration::ZERO);+ tokio::pin!(poll_deadline);let mut accepting_new_batches = true;-- while accepting_new_batches || !in_flight_polls.is_empty() {- self.drop_revoked_polls(&mut in_flight_polls);- // Consumer-level concurrency: how many Kafka batches are being- // processed in parallel, bounded by `max_in_flight_batches`.- gauge!("ingestion_consumer_in_flight_batches").set(in_flight_polls.len() as f64);-- // Not beside `collect_batch`: a tick there would drop a collection- // in progress.- if let Err(err) = self.commit_due() {- self.fail_commit(err);- return;- }-- let collect = accepting_new_batches- && in_flight_polls.len() < self.max_in_flight_batches- && (in_flight_polls.is_empty() || !self.every_partition_paused());- if collect {- tokio::select! {- _ = self.handle.shutdown_recv() => {- info!(- in_flight = in_flight_polls.len(),- "Shutdown signal received, draining in-flight batches"- );- accepting_new_batches = false;- self.batcher.begin_shutdown();- }- result = self.collect_batch() => {- let collected = match result {- Ok(collected) => collected,- Err(err) => {- self.fail_batch_processing(err);- return;- }- };-- if collected.accumulator.message_count() == 0 {- self.handle.report_healthy();- if in_flight_polls.is_empty() {- continue;- }- } else {- in_flight_polls.push_back(self.submit_poll(collected));- self.handle.report_healthy();-- if in_flight_polls.len() < self.max_in_flight_batches {- continue;- }- }+ loop {+ let mut submitted = false;+ while let Some(step) = poller.next_step(tokio::time::Instant::now()) {+ submitted |= matches!(step, Step::Submit(_));+ if let Err(failure) = self.carry_out(step, &mut poller) {+ match failure {+ Stopped::Drained => {}+ Stopped::Commit(err) => self.fail_commit(err),+ Stopped::Batch(err) => self.fail_batch_processing(err),}+ return;}}+ // rdkafka's stream spends none of tokio's budget, so a ready+ // backlog keeps `recv` ready on every turn and this loop would+ // never yield. The batcher runs on this runtime.+ if submitted {+ tokio::task::yield_now().await;+ }- if let Err(failure) = self- .complete_oldest_poll(&mut in_flight_polls, &mut completions, &mut errors)- .await- {- match failure {- Failure::Batch(err) => self.fail_batch_processing(err),- Failure::Commit(err) => self.fail_commit(err),+ let deadline = poller.deadline();+ if let Some(deadline) = deadline {+ if poll_deadline.deadline() != deadline {+ poll_deadline.as_mut().reset(deadline);+ }+ }+ let reading = poller.reading();+ let woke = tokio::select! {+ _ = self.handle.shutdown_recv(), if accepting_new_batches => Woke::Shutdown,+ completion = completions.recv() => Woke::Completion(completion),+ failure = errors.recv() => Woke::Error(failure),+ _ = tick.tick() => Woke::Tick,+ _ = &mut poll_deadline, if deadline.is_some() => Woke::Deadline,+ received = self.consumer.recv(), if reading => Woke::Received(received),+ };+ // The rebalance callbacks run inside `recv`, also inside one the+ // select dropped, so the poller sees each revoke before it acts+ // on anything that woke the loop.+ self.hand_over_revokes(&mut poller);++ let failure = match woke {+ Woke::Shutdown => {+ info!("Shutdown signal received, draining in-flight batches");+ accepting_new_batches = false;+ self.batcher.begin_shutdown();+ poller.shutdown();+ None}+ Woke::Completion(Some(completion)) => poller.completed(&completion).err(),+ Woke::Completion(None) => {+ Some(anyhow::anyhow!("batcher completion channel closed"))+ }+ Woke::Error(Some(message)) => Some(anyhow::anyhow!(message)),+ Woke::Error(None) => Some(anyhow::anyhow!("batcher error channel closed")),+ Woke::Tick => {+ self.handle.report_healthy();+ poller.ticked();+ None+ }+ Woke::Deadline => None,+ Woke::Received(Ok(message)) => {+ poller.received(&message, tokio::time::Instant::now());+ None+ }+ Woke::Received(Err(err)) => self.recv_failed(err).err(),+ };+ if let Some(err) = failure {+ self.fail_batch_processing(err);return;}}}+ fn carry_out<P: PollingStrategy>(&self, step: Step, poller: &mut P) -> Result<(), Stopped> {+ match step {+ Step::Submit(collected) => {+ let poll = self.submit_poll(collected).map_err(Stopped::Batch)?;+ poller.submitted(poll);+ self.handle.report_healthy();+ Ok(())+ }+ Step::Commit(commit) => self.commit(commit).map_err(Stopped::Commit),+ Step::Stop => Err(Stopped::Drained),+ Step::Fail(err) => Err(Stopped::Batch(err)),+ }+ }++ fn hand_over_revokes<P: PollingStrategy>(&self, poller: &mut P) {+ let revoked = std::mem::take(&mut *self.revoked_partitions.lock().unwrap());+ if !revoked.is_empty() {+ poller.revoked(&revoked);+ }+ }++ fn recv_failed(&self, err: KafkaError) -> anyhow::Result<()> {+ warn!(error = %err, "Kafka recv error");+ counter!("ingestion_consumer_kafka_errors_total").increment(1);+ // A fatal client error (such as UnreleasedInstanceId from a+ // static-membership collision) permanently disables the+ // consumer. Propagate it so the process exits and Kubernetes+ // restarts the pod, instead of re-polling a dead client forever+ // while still reporting healthy.+ if let Some((code, reason)) = self.consumer.client().fatal_error() {+ anyhow::bail!("fatal Kafka client error ({code:?}): {reason}");+ }+ Ok(())+ }+/// Submit each partition's next-to-read offset to Kafka, asynchronously.fn commit_offsets(&self, offsets: &HashMap<TopicPartition, Offset>) -> anyhow::Result<()> {let mut tpl = TopicPartitionList::new();@@ -823,13 +498,38 @@ impl IngestionConsumer {Ok(())}- /// Submit one collected poll to the batcher and track it as in flight.- fn submit_poll(&self, collected: CollectedBatch) -> InFlightPoll {+ fn submit_poll(&self, collected: CollectedBatch) -> anyhow::Result<InFlightPoll> {let CollectedBatch {accumulator,- partitions,+ mut partitions,stats,} = collected;+ let mut uncommitted = HashMap::with_capacity(partitions.len());+ let mut generations = HashMap::with_capacity(partitions.len());+ // One ledger call per partition keeps the lock and the gauge labels+ // off the per-message path.+ for (topic_partition, partition) in &mut partitions {+ partition.generation = self.topic_offset_ledger.generation(topic_partition);+ generations.insert(Partition(topic_partition.partition), partition.generation);+ match self+ .settlement+ .charge(topic_partition, partition.generation, &partition.charges)+ {+ Ok(charged) => {+ if charged.pause.is_some() {+ self.pause_partition(topic_partition);+ }+ uncommitted.insert(topic_partition.clone(), charged.uncommitted);+ }+ Err(rejection) => report_rejection(+ "charge",+ topic_partition,+ rejection,+ RejectedSlice::charged(&partition.charges),+ )?,+ }+ }+let message_count = accumulator.message_count();let poll_id = make_batch_id();emit_poll_stats(@@ -841,7 +541,7 @@ impl IngestionConsumer {self.batch_size_bytes,);- let assignment_epoch = self.batcher.submit(accumulator);+ let assignment_epoch = self.batcher.submit(accumulator, &generations);record_if(&self.debug_recorder, || DebugEventKind::BatchDispatched {batch_id: poll_id.clone(),messages: message_count,@@ -856,176 +556,63 @@ impl IngestionConsumer {"Kafka batch dispatched");- InFlightPoll {+ Ok(InFlightPoll {poll_id,assignment_epoch,partitions,+ uncommitted,message_count: message_count as u32,covered: 0,accepted: 0,dispatched_at: Instant::now(),- }- }-- /// Wait for completions to cover the oldest in-flight poll, then retire- /// it. Completions for newer polls are credited while waiting.- async fn complete_oldest_poll(- &self,- in_flight_polls: &mut VecDeque<InFlightPoll>,- completions: &mut mpsc::UnboundedReceiver<GroupCompletion>,- errors: &mut mpsc::UnboundedReceiver<String>,- ) -> Result<(), Failure> {- let mut heartbeat = tokio::time::interval_at(- tokio::time::Instant::now() + self.commit_tick,- self.commit_tick,- );- loop {- // A rebalance can drop the front poll (its partitions were- // revoked and its purged messages will never complete), so- // re-resolve it every pass instead of waiting on it forever.- self.drop_revoked_polls(in_flight_polls);- let Some(front) = in_flight_polls.front() else {- return Ok(());- };- if front.is_complete() {- break;- }- tokio::select! {- completion = completions.recv() => match completion {- Some(completion) => {- apply_completion(in_flight_polls, &completion, |tp, generation, run| {- self.settlement.settle_run(tp, generation, run)- })- .map_err(Failure::Batch)?;- // A resumed partition can fill a poll, so collect instead of waiting here.- if self.commit_due().map_err(Failure::Commit)? {- return Ok(());- }- }- None => return Err(Failure::Batch(anyhow::anyhow!(- "batcher completion channel closed"- ))),- },- failure = errors.recv() => match failure {- Some(message) => return Err(Failure::Batch(anyhow::anyhow!(message))),- None => return Err(Failure::Batch(anyhow::anyhow!(- "batcher error channel closed"- ))),- },- _ = heartbeat.tick() => {- self.handle.report_healthy();- if self.commit_due().map_err(Failure::Commit)?- || (in_flight_polls.len() < self.max_in_flight_batches- && !self.every_partition_paused())- {- return Ok(());- }- }- }- }-- let poll = pop_committable(in_flight_polls).map_err(Failure::Batch)?;-- self.settlement- .settle_poll(&poll.charged_slices())- .map_err(Failure::Batch)?;- self.commit_due().map_err(Failure::Commit)?;- emit_latest_processed_timestamp_metrics(&poll.partitions, &self.group_id);- record_if(&self.debug_recorder, || DebugEventKind::PollCompleted {- batch_id: poll.poll_id.clone(),- accepted: poll.accepted,- duration_ms: poll.dispatched_at.elapsed().as_millis() as u64,- partitions: debug_partition_offsets(&poll.partitions),- });-- counter!("ingestion_consumer_messages_processed_total").increment(poll.accepted as u64);- counter!("ingestion_consumer_batches_processed_total").increment(1);- self.handle.report_healthy();-- Ok(())- }-- /// Drop in-flight polls holding a revoked partition. The key table- /// purges those partitions' queued messages, so such a poll can never be- /// covered; its offsets stay uncommitted and replay under the new- /// assignment, and its late completions are discarded as stale.- fn drop_revoked_polls(&self, in_flight_polls: &mut VecDeque<InFlightPoll>) {- let revoked: Vec<RevokedPartition> =- std::mem::take(&mut *self.revoked_partitions.lock().unwrap());- if revoked.is_empty() {- return;- }- let stripped = strip_revoked_partitions(in_flight_polls, &revoked);- if stripped > 0 {- counter!("ingestion_consumer_polls_dropped_on_revoke_total").increment(stripped);- info!(- stripped,- "Removed revoked partitions' messages from in-flight polls"- );- }- }-- /// Paused partitions fill no poll, so a collection would wait out the batch- /// timeout before the completions whose commits resume them.- fn every_partition_paused(&self) -> bool {- self.fetching.all_paused()+ })}fn pause_partition(&self, topic_partition: &TopicPartition) {let tpl = partition_list(topic_partition);- match self+ if let Err(err) = self.consumer.pause(&tpl).and_then(|()| partition_error(&tpl)){- Ok(()) => self.fetching.paused(topic_partition),- Err(err) => error!(+ error!(topic = %topic_partition.topic,partition = topic_partition.partition,error = %err,"Could not pause a partition at its budget"- ),+ );}}fn resume_partition(&self, topic_partition: &TopicPartition) {let tpl = partition_list(topic_partition);- match self+ if let Err(err) = self.consumer.resume(&tpl).and_then(|()| partition_error(&tpl)){- Ok(()) => self.fetching.resumed(topic_partition),- Err(err) => error!(+ error!(topic = %topic_partition.topic,partition = topic_partition.partition,error = %err,"Could not resume a partition under its budget"- ),- }- }-- /// Returns whether the commit resumed a partition.- fn commit_due(&self) -> anyhow::Result<bool> {- match self.settlement.take_due(Instant::now()) {- Some(commit) => self.commit(commit),- None => Ok(false),+ );}}- fn commit_all(&self) -> anyhow::Result<bool> {+ fn commit_all(&self) -> anyhow::Result<()> {match self.settlement.take_all() {Some(commit) => self.commit(commit),- None => Ok(false),+ None => Ok(()),}}- fn commit(&self, commit: Commit) -> anyhow::Result<bool> {+ fn commit(&self, commit: Commit) -> anyhow::Result<()> {self.commit_offsets(&commit.offsets)?;for topic_partition in &commit.resume {self.resume_partition(topic_partition);}- Ok(!commit.resume.is_empty())+ Ok(())}fn fail_batch_processing(&self, err: anyhow::Error) {@@ -1052,168 +639,21 @@ impl IngestionConsumer {self.handle.signal_failure(format!("Offset commit failed: {err:#}"));}+}- /// Collect messages from Kafka until the first of `batch_size` messages,- /// `batch_size_bytes` of payload (when enabled), or `batch_timeout`.- ///- /// The byte bound is checked at the top of the loop, where accumulated- /// bytes are those of messages already appended: a batch therefore always- /// carries at least one message — a single payload larger than the whole- /// bound still moves rather than wedging the partition — and overshoot is- /// at most one message, itself bounded by `fetch.message.max.bytes`.- async fn collect_batch(&self) -> anyhow::Result<CollectedBatch> {- let mut accumulator = Accumulator::default();- let mut partitions: HashMap<TopicPartition, PartitionDeliveries> = HashMap::new();- let mut stats = BatchStats::new();- let deadline = Instant::now() + self.batch_timeout;- let batch_start_ms = current_time_ms();-- let mut stream = self.consumer.stream();- let mut current_topic: Option<Arc<str>> = None;-- loop {- if accumulator.message_count() >= self.batch_size {- break;- }-- if self.batch_size_bytes > 0 && stats.total_bytes >= self.batch_size_bytes {- counter!("ingestion_consumer_batches_byte_capped_total").increment(1);- break;- }-- let remaining = deadline.saturating_duration_since(Instant::now());- if remaining.is_zero() {- break;- }-- let poll_wait = remaining.min(Duration::from_secs(10));- match tokio::time::timeout(poll_wait, stream.next()).await {- Ok(Some(Ok(borrowed_message))) => {- if current_topic.as_deref() != Some(borrowed_message.topic()) {- current_topic = Some(Arc::from(borrowed_message.topic()));- }- let topic = Arc::clone(current_topic.as_ref().expect("set above"));- let partition = borrowed_message.partition();- let offset = borrowed_message.offset();- let kafka_ts = borrowed_message.timestamp().to_millis().unwrap_or(0);-- let payload_bytes = borrowed_message.payload().map(|v| v.len()).unwrap_or(0);- stats.total_bytes += payload_bytes;-- let mut headers = HashMap::new();- if let Some(rdkafka_headers) = borrowed_message.headers() {- for i in 0..rdkafka_headers.count() {- let header = rdkafka_headers.get(i);- if let Some(value) = header.value {- if let Ok(value_str) = std::str::from_utf8(value) {- headers.insert(header.key.to_string(), value_str.to_string());- }- }- }- }-- let lag_ms = headers- .get("now")- .and_then(|v| parse_now_ms(v))- .map(|capture_ms| (batch_start_ms - capture_ms).max(0));- if let Some(lag_ms) = lag_ms {- stats.message_lags_ms.push((partition, lag_ms));- }-- let delivery = Delivery {- offset,- charge: message_charge(&borrowed_message),- kafka_ts,- lag_ms,- };- let key = TopicPartition::new(topic.to_string(), partition);- let generations_version = self.topic_offset_ledger.generations_version();- match partitions.get_mut(&key) {- Some(deliveries) => deliveries.record(- generations_version,- || self.topic_offset_ledger.generation(&key),- &delivery,- ),- None => {- let generation = self.topic_offset_ledger.generation(&key);- partitions.insert(- key,- PartitionDeliveries::new(- generation,- generations_version,- &delivery,- ),- );- }- }-- let serialized = SerializedKafkaMessage {- topic,- partition,- offset,- timestamp: kafka_ts,- key: borrowed_message- .key()- .and_then(|k| std::str::from_utf8(k).ok())- .map(|s| s.to_string()),- value: borrowed_message- .payload()- .and_then(|v| std::str::from_utf8(v).ok())- .map(|s| s.to_string()),- headers,- };-- accumulator.push(Partition(partition), serialized.into());- }- Ok(Some(Err(err))) => {- warn!(error = %err, "Kafka recv error");- counter!("ingestion_consumer_kafka_errors_total").increment(1);- // A fatal client error (such as UnreleasedInstanceId from a- // static-membership collision) permanently disables the- // consumer. Propagate it so the process exits and Kubernetes- // restarts the pod, instead of re-polling a dead client forever- // while still reporting healthy.- if let Some((code, reason)) = self.consumer.client().fatal_error() {- anyhow::bail!("fatal Kafka client error ({code:?}): {reason}");- }- break;- }- Ok(None) => break,- Err(_) => {- self.handle.report_healthy();- if Instant::now() >= deadline {- break;- }- }- }- }-- // One ledger call per partition keeps the lock and the gauge labels- // off the per-message path.- for (topic_partition, partition) in &partitions {- // The ledger counts a rejection; it is logged here, where the- // slice is still known.- match self- .settlement- .charge(topic_partition, partition.generation, &partition.charges)- {- Ok(Some(Pause)) => self.pause_partition(topic_partition),- Ok(None) => {}- Err(rejection) => report_rejection(- "charge",- topic_partition,- rejection,- RejectedSlice::charged(&partition.charges),- )?,- }- }+enum Woke<'a> {+ Shutdown,+ Completion(Option<GroupCompletion>),+ Error(Option<String>),+ Tick,+ Deadline,+ Received(Result<BorrowedMessage<'a>, KafkaError>),+}- Ok(CollectedBatch {- accumulator,- partitions,- stats,- })- }+enum Stopped {+ Drained,+ Commit(anyhow::Error),+ Batch(anyhow::Error),}fn partition_list(topic_partition: &TopicPartition) -> TopicPartitionList {@@ -1222,9 +662,7 @@ fn partition_list(topic_partition: &TopicPartition) -> TopicPartitionList {tpl}-/// Emit the per-poll parity metrics (received counts, batch sizes,-/// utilization, and lag) right after collection, before the poll is-/// submitted.+/// Per-poll metrics, most with the same names as the Node.js consumer's.fn emit_poll_stats(stats: &BatchStats,partitions: &HashMap<TopicPartition, PartitionDeliveries>,@@ -1236,30 +674,25 @@ fn emit_poll_stats(counter!("ingestion_consumer_messages_received_total").increment(batch_size as u64);gauge!("ingestion_consumer_batch_size").set(batch_size as f64);- // Batch fill ratio (batch size / configured max) — matches Node.js- // `consumer_batch_utilization`. A useful scaling signal: sustained high- // utilization means batches are saturating and the consumer is demand-bound.+ // Matches Node.js `consumer_batch_utilization`.if max_batch_size > 0 {gauge!("consumer_batch_utilization", "groupId" => group_id.to_string()).set(batch_size as f64 / max_batch_size as f64);}- // The same ratio against the byte bound. Reported separately because the- // two disagree on lanes whose events are large: a count utilization can- // sit far below 1.0 while batches are in fact full, simply because the- // byte bound (or the prefetch queue behind it) ends collection first.- // Reading only the count ratio there invites raising a cap that cannot- // be reached. Absent when the byte bound is disabled.+ // The same ratio against the byte bound. On lanes with large events the+ // byte bound ends collection first, so the count ratio stays low while+ // batches are full. Absent when the byte bound is disabled.if max_batch_bytes > 0 {gauge!("consumer_batch_utilization_bytes", "groupId" => group_id.to_string()).set(stats.total_bytes as f64 / max_batch_bytes as f64);}- // Batch size distribution — matches Node.js `consumer_batch_size` histogram.+ // Matches Node.js.histogram!("consumer_batch_size").record(batch_size as f64);histogram!("consumer_batch_size_kb").record(stats.total_bytes as f64 / 1024.0);- // Per-partition ingestion lag gauge — matches Node.js `ingestion_lag_ms`.+ // Matches Node.js.for (topic_partition, partition) in partitions {let Some(max_lag) = partition.max_lag_ms else {continue;@@ -1273,7 +706,7 @@ fn emit_poll_stats(.set(max_lag as f64);}- // Per-message lag histogram — matches Node.js `ingestion_lag_ms_histogram`.+ // Matches Node.js.for (partition, lag_ms) in &stats.message_lags_ms {histogram!("ingestion_lag_ms_histogram",@@ -1284,143 +717,14 @@ fn emit_poll_stats(}}-/// Per-partition max offset + observed lag for the debug UI's batch events.-fn debug_partition_offsets(- partitions: &HashMap<TopicPartition, PartitionDeliveries>,-) -> Vec<PartitionOffset> {- partitions- .iter()- .map(|(topic_partition, partition)| PartitionOffset {- topic: topic_partition.topic.clone(),- partition: topic_partition.partition,- offset: partition.span.last,- lag_ms: partition.max_lag_ms.unwrap_or(0),- })- .collect()-}--/// Per-partition offset ranges, so the debug UI can match requests to polls.-fn debug_partition_spans(- partitions: &HashMap<TopicPartition, PartitionDeliveries>,-) -> Vec<PartitionOffsets> {- partitions- .iter()- .map(|(topic_partition, partition)| PartitionOffsets {- topic: topic_partition.topic.clone(),- partition: topic_partition.partition,- ranges: vec![(partition.span.first, partition.span.last)],- })- .collect()-}--/// One delivered message's cost against the ledger: one event, and the bytes-/// the process holds for the message. The byte count is a memory bound, so it-/// is the payload plus the key plus every header key and value, not only the-/// payload.-fn message_charge(message: &impl Message) -> Charge {- let payload_bytes = message.payload().map(|v| v.len()).unwrap_or(0);- let key_bytes = message.key().map(|k| k.len()).unwrap_or(0);- let mut header_bytes = 0;- if let Some(headers) = message.headers() {- for i in 0..headers.count() {- let header = headers.get(i);- header_bytes += header.key.len() + header.value.map(|v| v.len()).unwrap_or(0);- }- }- Charge {- events: 1,- bytes: (payload_bytes + key_bytes + header_bytes) as u64,- }-}--fn current_time_ms() -> i64 {- std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .unwrap_or_default()- .as_millis() as i64-}--/// Parse an ISO 8601 / RFC 3339 timestamp string into milliseconds since epoch.-/// Returns `None` if the string is missing or unparseable.-fn parse_now_ms(value: &str) -> Option<i64> {- chrono::DateTime::parse_from_rfc3339(value)- .ok()- .map(|dt| dt.timestamp_millis())-}--fn emit_latest_processed_timestamp_metrics(- partitions: &HashMap<TopicPartition, PartitionDeliveries>,- group_id: &str,-) {- // Same name as the Node.js consumer's gauge, so one dashboard reads both.- for (topic_partition, partition) in partitions {- gauge!(- "latest_processed_timestamp_ms",- "topic" => topic_partition.topic.clone(),- "partition" => topic_partition.partition.to_string(),- "groupId" => group_id.to_string()- )- .set(partition.latest_kafka_ts as f64);- }-}-#[cfg(test)]mod tests {use super::*;use std::num::NonZeroU64;- use crate::settlement::{charge_offsets, GroupSettlement};- use common_kafka_consumer::Offset as MessageOffset;-- fn poll(epoch: u64, partition: i32, first: i64, last: i64, count: u32) -> InFlightPoll {- let mut partitions = HashMap::new();- partitions.insert(- TopicPartition::new("test", partition),- PartitionDeliveries {- span: OffsetSpan { first, last },- generation: 0,- generations_version_seen: 0,- charges: (first..=last)- .map(|offset| (MessageOffset(offset), Charge::ZERO))- .collect(),- latest_kafka_ts: 0,- max_lag_ms: None,- delivered: count,- covered: 0,- accepted: 0,- credited: HashSet::default(),- },- );- InFlightPoll {- poll_id: format!("poll-{epoch}-{partition}-{first}"),- assignment_epoch: epoch,- partitions,- message_count: count,- covered: 0,- accepted: 0,- dispatched_at: Instant::now(),- }- }-- fn completion(epoch: u64, partition: i32, offsets: &[i64], accepted: u32) -> GroupCompletion {- GroupCompletion {- partition: Partition(partition),- assignment_epoch: epoch,- offsets: offsets.iter().map(|o| MessageOffset(*o)).collect(),- accepted,- }- }+ use common_kafka_consumer::{Charge, Offset as MessageOffset};- fn credit(in_flight: &mut VecDeque<InFlightPoll>, completion: GroupCompletion) {- apply_completion(in_flight, &completion, |_, _, _| Ok(())).unwrap();- }-- fn partitions(numbers: &[i32]) -> Vec<TopicPartition> {- numbers- .iter()- .map(|&number| TopicPartition::new("events", number))- .collect()- }+ use crate::settlement::{GroupSettlement, Pause};#[rstest::rstest]#[case::group_below_a_second(CompletionGranularity::Group, 200, 200)]@@ -1438,49 +742,6 @@ mod tests {);}- #[test]- fn every_partition_is_paused_only_while_none_is_fetching() {- let fetching = FetchingPartitions::default();- assert!(!fetching.all_paused(), "nothing is assigned");-- fetching.assigned(&partitions(&[0, 1]));- fetching.paused(&TopicPartition::new("events", 0));- assert!(!fetching.all_paused(), "partition 1 still fetches");-- fetching.paused(&TopicPartition::new("events", 1));- assert!(fetching.all_paused());-- fetching.resumed(&TopicPartition::new("events", 1));- assert!(!fetching.all_paused());- }-- #[test]- fn a_rebalance_changes_which_partitions_must_be_paused() {- let fetching = FetchingPartitions::default();- fetching.assigned(&partitions(&[0, 1]));- fetching.paused(&TopicPartition::new("events", 0));-- fetching.revoked(&partitions(&[1]));- assert!(fetching.all_paused(), "only the paused partition is left");-- fetching.assigned(&partitions(&[0]));- assert!(- !fetching.all_paused(),- "a reassigned partition starts fetching"- );- }-- #[test]- fn a_partition_that_is_not_assigned_neither_pauses_nor_resumes() {- let fetching = FetchingPartitions::default();-- fetching.paused(&TopicPartition::new("events", 0));- assert!(!fetching.all_paused());-- fetching.resumed(&TopicPartition::new("events", 0));- assert!(!fetching.all_paused());- }-#[test]fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {let ledger = Arc::new(TopicOffsetLedger::new());@@ -1490,10 +751,11 @@ mod tests {BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),));let epoch = AssignmentEpoch::new();+ let revoked_partitions = Arc::new(Mutex::new(Vec::new()));let on_assign = assign_hook(Arc::clone(&settlement),- Arc::new(FetchingPartitions::default()),epoch.clone(),+ Arc::clone(&revoked_partitions),);let p0 = TopicPartition::new("events", 0);let one_event = [(@@ -1503,7 +765,12 @@ mod tests {bytes: 0,},)];- assert_eq!(settlement.charge(&p0, 0, &one_event), Ok(Some(Pause)));+ assert_eq!(+ settlement+ .charge(&p0, 0, &one_event)+ .map(|charged| charged.pause),+ Ok(Some(Pause))+ );settlement.settle_run(&p0, 0, &[MessageOffset(10)]).unwrap();on_assign(std::slice::from_ref(&p0));@@ -1512,426 +779,16 @@ mod tests {assert_eq!(ledger.generation(&p0), 1);assert_eq!(epoch.current(), 2);assert_eq!(- settlement.charge(&p0, 1, &one_event),- Ok(Some(Pause)),- "the regained partition starts unpaused"- );- }-- fn regained_poll(charged: i64, span_last: i64, delivered: u32) -> InFlightPoll {- let mut poll = poll(1, 0, charged, charged, delivered);- let deliveries = poll- .partitions- .get_mut(&TopicPartition::new("test", 0))- .unwrap();- deliveries.span.last = span_last;- deliveries.generation = 2;- poll- }-- #[test]- fn an_accepted_completion_settles_one_run_per_poll() {- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);- in_flight[1]- .partitions- .get_mut(&TopicPartition::new("test", 0))- .unwrap()- .generation = 1;- let mut runs = Vec::new();-- apply_completion(- &mut in_flight,- &completion(1, 0, &[2, 3, 4, 5], 4),- |topic_partition, generation, offsets| {- runs.push((topic_partition.partition, generation, offsets.to_vec()));- Ok(())- },- )- .unwrap();-- assert_eq!(- runs,- vec![- (0, 0, vec![MessageOffset(2), MessageOffset(3)]),- (0, 1, vec![MessageOffset(4), MessageOffset(5)]),- ],- "each run carries its own poll's generation"- );- }-- #[test]- fn a_completion_spanning_two_polls_commits_once_the_prefix_is_whole() {- let ledger = Arc::new(TopicOffsetLedger::new());- let settlement =- GroupSettlement::new(Arc::clone(&ledger), Duration::ZERO, BudgetLimits::unset());- let partition = TopicPartition::new("test", 0);- charge_offsets(&ledger, &partition, 0, &[0, 1, 2, 3, 4, 5, 6, 7]);- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);- let mut settle = |topic_partition: &TopicPartition, generation: u64, offsets: &[Offset]| {- settlement.settle_run(topic_partition, generation, offsets)- };-- apply_completion(- &mut in_flight,- &completion(1, 0, &[2, 3, 4, 5], 4),- &mut settle,- )- .unwrap();- assert!(- settlement.take_due(Instant::now()).is_none(),- "offsets 0 and 1 are still in flight"- );-- apply_completion(- &mut in_flight,- &completion(1, 0, &[0, 1, 6, 7], 4),- &mut settle,- )- .unwrap();- assert_eq!(- settlement- .take_due(Instant::now())- .map(|commit| commit.offsets),- Some(HashMap::from([(partition, MessageOffset(8))]))- );- }-- #[test]- fn a_short_accepted_completion_settles_nothing() {- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4)]);- let mut runs = 0;-- apply_completion(&mut in_flight, &completion(1, 0, &[0, 1], 1), |_, _, _| {- runs += 1;- Ok(())- })- .unwrap();-- assert_eq!(runs, 0, "the ledger must not commit past a dropped message");- assert_eq!(in_flight[0].covered, 2);- assert_eq!(in_flight[0].accepted, 1);- }-- #[test]- fn a_regained_poll_settles_each_charged_offset_once() {- // Offsets 8..10, a regain, then offset 8 again: four deliveries, one charge.- let mut in_flight = VecDeque::from([regained_poll(8, 10, 4)]);- let mut runs = Vec::new();- let mut settle = |_: &TopicPartition, generation: u64, offsets: &[Offset]| {- runs.push((generation, offsets.to_vec()));- Ok(())- };-- apply_completion(- &mut in_flight,- &completion(1, 0, &[8, 9, 10], 3),- &mut settle,- )- .unwrap();- apply_completion(&mut in_flight, &completion(1, 0, &[8], 1), &mut settle).unwrap();-- assert_eq!(- runs,- vec![(2, vec![MessageOffset(8)])],- "neither the uncharged old offsets nor the redelivered one reach the ledger"- );- assert!(in_flight[0].is_complete());- assert!(pop_committable(&mut in_flight).is_ok());- }-- #[test]- fn apply_completion_spanning_two_polls_credits_each() {- // A key-table run merges messages from consecutive polls into one- // send, so its completion spans both spans.- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);-- credit(&mut in_flight, completion(1, 0, &[2, 3, 4, 5], 4));-- assert_eq!(in_flight[0].covered, 2);- assert_eq!(in_flight[0].accepted, 2);- assert_eq!(in_flight[1].covered, 2);- assert_eq!(in_flight[1].accepted, 2);- }-- #[test]- fn apply_completion_under_report_shorts_the_tail_poll() {- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);-- credit(&mut in_flight, completion(1, 0, &[2, 3, 4, 5], 3));-- assert_eq!(in_flight[0].accepted, 2, "leading offsets are accepted");- assert_eq!(- in_flight[1].accepted, 1,- "the tail poll fails its accepted check"- );- assert_eq!(in_flight[1].covered, 2);- }-- #[test]- fn a_complete_poll_with_a_charged_offset_no_completion_covered_is_not_committable() {- let mut in_flight = VecDeque::from([poll(1, 0, 10, 12, 3)]);-- credit(&mut in_flight, completion(1, 0, &[10, 12, 12], 3));-- assert!(in_flight[0].is_complete());- let error = pop_committable(&mut in_flight)- .err()- .expect("offset 11 has no completion");- assert!(error.to_string().contains("offset 11 "), "{error}");- }-- #[test]- fn a_poll_whose_deliveries_span_an_offset_gap_commits() {- // A compacted or transactional topic skips offsets, so a span can far- // exceed the poll's message count.- let mut gapped = poll(1, 0, 0, 0, 2);- let deliveries = gapped- .partitions- .get_mut(&TopicPartition::new("test", 0))- .unwrap();- deliveries.span.last = i64::MAX;- deliveries- .charges- .push((MessageOffset(i64::MAX), Charge::ZERO));- let mut in_flight = VecDeque::from([gapped]);-- credit(&mut in_flight, completion(1, 0, &[0, i64::MAX], 2));-- assert!(in_flight[0].is_complete());- assert!(pop_committable(&mut in_flight).is_ok());- }-- #[test]- fn strip_revoked_partitions_keeps_the_other_partitions_slices() {- // A cooperative rebalance revokes one partition of a two-partition- // poll. The kept partition's slice must stay accounted, or its- // ledger charges never settle and its commits freeze at the hole.- let mut two = poll(1, 0, 0, 1, 4);- two.partitions.insert(- TopicPartition::new("test", 1),- PartitionDeliveries {- span: OffsetSpan {- first: 10,- last: 11,- },- generation: 0,- generations_version_seen: 0,- charges: Vec::new(),- latest_kafka_ts: 0,- max_lag_ms: None,- delivered: 2,- covered: 0,- accepted: 0,- credited: HashSet::default(),- },- );- two.partitions- .get_mut(&TopicPartition::new("test", 0))- .unwrap()- .delivered = 2;- let mut in_flight = VecDeque::from([two]);- // Partition 1 already had one message covered before the revoke.- credit(&mut in_flight, completion(1, 1, &[10], 1));-- let stripped = strip_revoked_partitions(- &mut in_flight,- &[RevokedPartition {- topic_partition: TopicPartition::new("test", 1),- generation: 1,- }],- );-- assert_eq!(stripped, 2);- assert_eq!(in_flight[0].message_count, 2);- assert_eq!(in_flight[0].covered, 0, "the revoked slice's credit goes");- assert!(!in_flight[0].is_complete());-- // The kept partition completes the poll; a late completion for the- // revoked partition is discarded, not credited.- credit(&mut in_flight, completion(1, 1, &[11], 1));- assert_eq!(in_flight[0].covered, 0);- credit(&mut in_flight, completion(1, 0, &[0, 1], 2));- assert!(in_flight[0].is_complete());- assert_eq!(in_flight[0].accepted, 2);- assert!(pop_committable(&mut in_flight).is_ok());- }-- #[test]- fn strip_revoked_partitions_removes_an_emptied_poll() {- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 2, 0, 3, 4)]);-- let stripped = strip_revoked_partitions(- &mut in_flight,- &[RevokedPartition {- topic_partition: TopicPartition::new("test", 0),- generation: 1,- }],- );-- assert_eq!(stripped, 4);- assert_eq!(in_flight.len(), 1);- assert_eq!(in_flight[0].message_count, 4);- }-- #[test]- fn historical_revoke_keeps_a_reassigned_poll_and_its_ledger_slice() {- let topic_partition = TopicPartition::new("test", 0);- let ledger = TopicOffsetLedger::new();- let delivery = Delivery {- offset: 10,- charge: Charge {- events: 1,- bytes: 1,- },- kafka_ts: 0,- lag_ms: None,- };-- // This collection saw offset 10 immediately before a revoke, then- // saw its replay after the partition was assigned again. Only the- // current generation is charged, while the poll still covers both- // deliveries that the batcher will complete.- let mut reassigned_deliveries = PartitionDeliveries::new(- ledger.generation(&topic_partition),- ledger.generations_version(),- &delivery,- );- ledger.forget_partitions([("test", 0)]);- ledger.forget_partitions([("test", 0)]);- reassigned_deliveries.record(- ledger.generations_version(),- || ledger.generation(&topic_partition),- &delivery,- );- assert_eq!(reassigned_deliveries.generation, 2);- assert_eq!(reassigned_deliveries.delivered, 2);- assert_eq!(reassigned_deliveries.charges.len(), 1);- ledger- .charge(- &topic_partition,- reassigned_deliveries.generation,- reassigned_deliveries.charges.iter().copied(),- )- .unwrap();-- // The queued revoke notification belongs to generation 0. It must- // remove genuine old in-flight work, but not this poll collected and- // charged under the later reassignment.- let old_poll = poll(0, 0, 1, 1, 1);- let reassigned_poll = InFlightPoll {- poll_id: "reassigned".to_string(),- assignment_epoch: 1,- partitions: HashMap::from([(topic_partition.clone(), reassigned_deliveries)]),- message_count: 2,- covered: 0,- accepted: 0,- dispatched_at: Instant::now(),- };- let mut in_flight = VecDeque::from([old_poll, reassigned_poll]);-- let stripped = strip_revoked_partitions(- &mut in_flight,- &[RevokedPartition {- topic_partition: topic_partition.clone(),- generation: 1,- }],- );-- assert_eq!(stripped, 1, "only the old assignment is stripped");- assert_eq!(in_flight.len(), 1);- assert_eq!(in_flight[0].poll_id, "reassigned");-- credit(&mut in_flight, completion(1, 0, &[10, 10], 2));- assert!(in_flight[0].is_complete());- let reassigned = pop_committable(&mut in_flight)- .expect("an offset credited twice still counts as credited");- let retained = reassigned.partitions.get(&topic_partition).unwrap();- ledger- .settle(- &topic_partition,- retained.generation,- retained.charges.iter().map(|(offset, _)| *offset),- )- .unwrap();- assert_eq!(- ledger- .take_frontier(&topic_partition)- .map(|taken| taken.offset),- Some(Offset(11))+ *revoked_partitions.lock().unwrap(),+ vec![p0.clone()],+ "the poller drops what it buffered from the lost assignment");-- // With the retained replay slice settled, the next accepted offset- // advances normally instead of remaining stuck behind offset 10.- ledger- .charge(- &topic_partition,- retained.generation,- [(Offset(11), delivery.charge)],- )- .unwrap();- ledger- .settle(&topic_partition, retained.generation, [Offset(11)])- .unwrap();- assert_eq!(- ledger- .take_frontier(&topic_partition)- .map(|taken| taken.offset),- Some(Offset(12))- );- assert_eq!(ledger.held(&topic_partition).offsets, 0);- }-- #[test]- fn apply_completion_requires_a_matching_epoch() {- // The same offsets exist in two polls when a partition was revoked,- // reassigned, and replayed. The epoch keeps each incarnation's- // completions in its own poll.- let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(2, 0, 0, 3, 4)]);-- credit(&mut in_flight, completion(2, 0, &[0, 1, 2, 3], 4));-- assert_eq!(in_flight[0].covered, 0);- assert_eq!(in_flight[1].covered, 4);- }-- #[test]- fn a_covered_head_is_popped_before_a_redelivered_offset_is_matched() {- let ledger = Arc::new(TopicOffsetLedger::new());- let settlement =- GroupSettlement::new(Arc::clone(&ledger), Duration::ZERO, BudgetLimits::unset());- let partition = TopicPartition::new("test", 0);- ledger.forget_partitions([("test", 0)]);- charge_offsets(&ledger, &partition, 1, &[0, 1]);- let mut settle = |topic_partition: &TopicPartition, generation: u64, offsets: &[Offset]| {- settlement.settle_run(topic_partition, generation, offsets)- };-- // Both polls carry the same offsets: a regain during collection.- let mut old = poll(3, 0, 0, 1, 2);- old.poll_id = "old".into();- let mut replay = poll(3, 0, 0, 1, 2);- replay.poll_id = "replay".into();- replay.partitions.get_mut(&partition).unwrap().generation = 1;- let mut in_flight = VecDeque::from([old, replay]);- let event = completion(3, 0, &[0, 1], 2);-- apply_completion(&mut in_flight, &event, &mut settle).unwrap();- assert_eq!(in_flight[0].poll_id, "old");- assert!(in_flight[0].is_complete());- assert!(- settlement.take_due(Instant::now()).is_none(),- "the old generation's settle is stale"- );- in_flight.pop_front();-- apply_completion(&mut in_flight, &event, &mut settle).unwrap();- assert_eq!(in_flight[0].poll_id, "replay");- assert!(in_flight[0].is_complete());assert_eq!(settlement- .take_due(Instant::now())- .map(|commit| commit.offsets),- Some(HashMap::from([(partition, MessageOffset(2))]))+ .charge(&p0, 1, &one_event)+ .map(|charged| charged.pause),+ Ok(Some(Pause)),+ "the regained partition starts unpaused");}}diff --git a/rust/ingestion-consumer/src/lib.rs b/rust/ingestion-consumer/src/lib.rsindex 81893274b8ce..97d276af1a04 100644--- a/rust/ingestion-consumer/src/lib.rs+++ b/rust/ingestion-consumer/src/lib.rs@@ -1,5 +1,6 @@pub mod aperture;pub mod batcher;+pub mod charge_limit;mod commit_pacer;pub mod config;pub mod consumer;@@ -10,6 +11,7 @@ pub mod kafka_stats;pub mod ledger_rejection;pub mod order_sentinel;pub mod partition_budget;+mod polling;pub mod readiness;pub mod rebalance;pub mod routing;diff --git a/rust/ingestion-consumer/src/order_sentinel.rs b/rust/ingestion-consumer/src/order_sentinel.rsindex faf9999a339d..c9a7904aaaba 100644--- a/rust/ingestion-consumer/src/order_sentinel.rs+++ b/rust/ingestion-consumer/src/order_sentinel.rs@@ -1,20 +1,12 @@//! The per-key send-order sentinel.//!-//! **Per-key send order** ([`KeyOrderSentinel`]): for every routing key-//! (the Kafka message key), messages must be handed to workers in Kafka offset-//! order, and a message must never be re-sent after it was ACKed. Replays of-//! un-ACKed messages on the retry paths ([`SendKind::Resend`]: a failed or-//! partially acked send's replay) are legal at-least-once behavior and are-//! counted separately (`ingestion_consumer_key_replays_total`) rather than-//! flagged; the same regression on a fresh send is a `send_below_last_sent`-//! violation. Checked by the batcher as it begins each send, the point that-//! defines the intended per-key order. Only messages produced with a Kafka key participate: null-key production-//! (e.g. overflow rerouting) spreads a routing key across partitions,-//! deliberately forfeiting per-key order, so there is no invariant to check-//! and offsets from different partitions are not comparable. Skipped messages-//! are counted in `ingestion_consumer_key_sentinel_unkeyed_total`.+//! For every routing key (the Kafka message key), messages must go to+//! workers in offset order, and an ACKed message must never be sent again.+//! [`SendKind::Resend`] may repeat un-ACKed offsets. The batcher checks each+//! send as it begins it. Null-key messages are skipped: null-key production+//! spreads a key across partitions, so it has no per-key order.//!-//! The sentinel is a pure observer: it never influences routing.+//! The sentinel only observes: it never changes routing.use std::collections::HashMap;use std::sync::atomic::{AtomicBool, Ordering};@@ -40,14 +32,12 @@ impl OffsetSpan {}}- /// Widen the span to include `offset`.pub fn extend(&mut self, offset: i64) {self.first = self.first.min(offset);self.last = self.last.max(offset);}}-/// How a send violated the per-key order invariant.#[derive(Clone, Copy, Debug, PartialEq, Eq)]pub enum KeyOrderViolationKind {/// Offsets within one assigned group were not strictly ascending.@@ -83,7 +73,6 @@ pub enum SendKind {Resend,}-/// One detected per-key order violation, returned for tests and logged.#[derive(Clone, Debug, PartialEq, Eq)]pub struct KeyOrderViolation {pub kind: KeyOrderViolationKind,@@ -112,8 +101,6 @@ pub struct KeyOrderSentinel {/// Keyed by customer-chosen routing keys, so the hasher is seeded per/// map to resist collision flooding.keys: Mutex<HashMap<Arc<str>, KeyState, ahash::RandomState>>,- /// Kill switch (`CONSUMER_ORDER_SENTINEL_ENABLED`). When off, checks- /// no-op and no state accumulates.enabled: AtomicBool,}@@ -158,7 +145,6 @@ impl KeyOrderSentinel {gauge!("ingestion_consumer_key_sentinel_keys").set(0.0);}- /// Number of tracked keys (bounded by in-flight work; exposed for tests).pub fn key_count(&self) -> usize {self.keys.lock().unwrap().len()}@@ -206,12 +192,9 @@ pub struct SentinelBatch<'a> {}impl SentinelBatch<'_> {- /// Record that `messages` for `routing_key` are being handed to a worker.- /// `kind` distinguishes a fresh assignment (offsets must only move- /// forward) from a retry-path resend (repeating un-ACKed offsets is- /// legal). Call in the intended per-key send order. Null-key messages are- /// skipped — they carry no per-key order promise (see module docs). Emits- /// metrics and logs; returns violations for tests.+ /// Record that `messages` for `routing_key` go to a worker now. Call in the+ /// intended per-key send order. Skips null-key messages. Emits metrics and+ /// logs; returns the violations for tests.pub fn note_sent(&mut self,key: &(impl SentinelKey + ?Sized),@@ -285,7 +268,6 @@ impl SentinelBatch<'_> {last_acked: None,};} else if first.offset > state.last_sent {- // Normal forward progress.state.last_sent = last_sent;} else if state.last_acked.is_some_and(|acked| first.offset <= acked) {violations.push(KeyOrderViolation {@@ -296,16 +278,10 @@ impl SentinelBatch<'_> {});state.last_sent = state.last_sent.max(last_sent);} else if kind == SendKind::Resend {- // Replay of a not-yet-ACKed range: the legal retry path- // (send failure → defer → flush re-routes the same messages).counter!("ingestion_consumer_key_replays_total").increment(1);state.last_sent = state.last_sent.max(last_sent);} else {- // A fresh send regressed: a newer batch's assignment for- // this key overtook an older batch's. (A rebalance racing- // an in-flight batch can produce a rare false positive- // until epoch-scoped baselines land — treat as near-zero,- // not hard-zero.)+ // A fresh send regressed: a newer send for this key overtook an older one.violations.push(KeyOrderViolation {kind: KeyOrderViolationKind::SendBelowLastSent,routing_key: routing_key.to_string(),@@ -335,9 +311,8 @@ impl SentinelBatch<'_> {violations}- /// Record that a worker ACKed this key's messages up to `max_offset`.- /// ACKs may arrive out of order across concurrent sub-batches (HTTP- /// completion order), so this only ever advances the high-water mark.+ /// Record that a worker ACKed this key's messages up to `max_offset`. ACKs+ /// from different workers can arrive out of order, so the mark only advances.pub fn note_acked(&mut self, routing_key: &str, max_offset: i64) {let Some(keys) = self.keys.as_mut() else {return;@@ -408,6 +383,7 @@ mod tests {key: Some("t:a".to_string()),value: None,headers: HashMap::new(),+ generation: 0,}}@@ -418,8 +394,6 @@ mod tests {}}- // ---- KeyOrderSentinel -----#[test]fn forward_sends_pass() {let sentinel = KeyOrderSentinel::new();@@ -453,7 +427,6 @@ mod tests {sentinel.batch().note_sent("t:a", &[msg_at(0, 1), msg_at(0, 2)], SendKind::Fresh);- // Send failed (no ACK) → the replay re-sends the same messages.assert!(sentinel.batch().note_sent("t:a", &[msg_at(0, 1), msg_at(0, 2)], SendKind::Resend)@@ -462,9 +435,6 @@ mod tests {#[test]fn fresh_send_below_last_sent_is_a_violation() {- // A fresh assignment regressing below the key's sent watermark means a- // newer batch's send overtook an older batch's — the race the- // consumer-loop assignment ordering exists to prevent.let sentinel = KeyOrderSentinel::new();sentinel.batch()@@ -520,8 +490,7 @@ mod tests {#[test]fn a_redelivered_offset_keeps_the_ack_mark_below_every_unfinished_message() {- // The run sent offset 1, 2, then 1 again, and the worker accepted the- // first copy only, so the unfinished list runs 2 then 1.+ // Offset 1 was sent twice; the worker accepted the first copy only.assert_eq!(ack_mark(0, [(0, 1, true)], [(0, 2), (0, 1)]), None);}@@ -532,9 +501,6 @@ mod tests {#[test]fn older_messages_after_acked_newer_ones_are_a_violation() {- // The exact race the deferral machinery exists to prevent: a key's- // newer messages were sent and ACKed while its older ones were still- // deferred — flushing the older ones now is out-of-order processing.let sentinel = KeyOrderSentinel::new();sentinel.batch()@@ -557,7 +523,6 @@ mod tests {sentinel.batch().note_sent("t:a", &[msg_at(0, 3), msg_at(0, 4)], SendKind::Fresh);- // Sub-batch ACKs arrive in reverse HTTP-completion order.sentinel.batch().note_acked("t:a", 4);sentinel.batch().note_acked("t:a", 2);// Forward progress from the true high-water mark is still clean.@@ -565,7 +530,6 @@ mod tests {.batch().note_sent("t:a", &[msg_at(0, 5)], SendKind::Fresh).is_empty());- // …and re-sending below it still fires.let violations = sentinel.batch().note_sent("t:a", &[msg_at(0, 3)], SendKind::Resend);@@ -582,7 +546,6 @@ mod tests {sentinel.batch().note_acked("t:a", 5);sentinel.batch().evict("t:a");assert_eq!(sentinel.key_count(), 0);- // A rebaselined key doesn't compare against evicted history.assert!(sentinel.batch().note_sent("t:a", &[msg_at(0, 6)], SendKind::Fresh)@@ -628,7 +591,6 @@ mod tests {keys.batch().note_acked("t:a", 5);keys.set_enabled(false);keys.set_enabled(true);- // Re-enable rebaselines: no comparison against pre-disable history.assert!(keys.batch().note_sent("t:a", &[msg_at(0, 3)], SendKind::Fresh)@@ -664,8 +626,7 @@ mod tests {#[test]fn null_key_messages_are_ignored() {let sentinel = KeyOrderSentinel::new();- // Null-key production round-robins a key across partitions; there is- // no per-key order to check, even when offsets regress across sends.+ // Null-key messages have no per-key order, so regressing offsets do not fire.assert!(sentinel.batch().note_sent("t:a", &[unkeyed_msg_at(1, 5000)], SendKind::Fresh)@@ -679,10 +640,7 @@ mod tests {#[test]fn null_key_offsets_do_not_inflate_watermarks() {- // A mixed group (keyed traffic that overflowed mid-stream): the- // unkeyed message's offset comes from another partition and must not- // advance the key's send/ACK watermarks — otherwise the next keyed- // send would fire a false resend_after_ack.+ // An unkeyed offset must not raise the key's watermarks.let sentinel = KeyOrderSentinel::new();assert!(sentinel.batch()diff --git a/rust/ingestion-consumer/src/partition_budget.rs b/rust/ingestion-consumer/src/partition_budget.rsindex e735fcf03cf1..f5d1439c8fae 100644--- a/rust/ingestion-consumer/src/partition_budget.rs+++ b/rust/ingestion-consumer/src/partition_budget.rs@@ -2,31 +2,12 @@ use std::num::NonZeroU64;use common_kafka_consumer::Charge;-#[derive(Debug, Clone, Copy)]-struct Threshold {- events: Option<u64>,- bytes: Option<u64>,-}--impl Threshold {- fn reached(&self, charge: Charge) -> bool {- self.events.is_some_and(|limit| charge.events >= limit)- || self.bytes.is_some_and(|limit| charge.bytes >= limit)- }-- fn scaled(&self, fraction: f64) -> Self {- let scale = |limit: u64| ((limit as f64 * fraction) as u64).clamp(1, limit);- Self {- events: self.events.map(scale),- bytes: self.bytes.map(scale),- }- }-}+use crate::charge_limit::ChargeLimit;#[derive(Debug, Clone, Copy)]struct Limits {- pause_at: Threshold,- resume_at: Threshold,+ pause_at: ChargeLimit,+ resume_at: ChargeLimit,}#[derive(Debug, Clone, Copy)]@@ -50,13 +31,10 @@ impl BudgetLimits {"the low watermark must be above 0 and at most 1, got {low_watermark}"));}- if events.is_none() && bytes.is_none() {+ let pause_at = ChargeLimit::new(events, bytes);+ if !pause_at.is_set() {return Ok(Self::unset());}- let pause_at = Threshold {- events: events.map(NonZeroU64::get),- bytes: bytes.map(NonZeroU64::get),- };Ok(Self {limits: Some(Limits {pause_at,@@ -69,6 +47,11 @@ impl BudgetLimits {self.limits.is_some()}+ pub fn sets_both_axes(&self) -> bool {+ self.limits+ .is_some_and(|limits| limits.pause_at.sets_both_axes())+ }+pub(crate) fn pauses(&self, uncommitted: Charge) -> bool {self.limits.is_some_and(|limits| limits.pause_at.reached(uncommitted))diff --git a/rust/ingestion-consumer/src/polling/capped.rs b/rust/ingestion-consumer/src/polling/capped.rsnew file mode 100644index 000000000000..313ab5fa0226--- /dev/null+++ b/rust/ingestion-consumer/src/polling/capped.rs@@ -0,0 +1,545 @@+use std::collections::{HashMap, VecDeque};+use std::sync::Arc;++use common_kafka_consumer::{GroupCompletion, Offset, TopicPartition};+use metrics::{counter, gauge};+use rdkafka::message::Message;+use tokio::time::Instant;+use tracing::warn;++use super::poll::{debug_partition_offsets, InFlightPoll, PartitionDeliveries};+use super::{report_revoked, Collector, Parts, PollingStrategy, Step};+use crate::debug_recorder::{record_if, DebugEventKind, DebugRecorder};+use crate::settlement::Settlement;++/// Admits up to `max_in_flight_polls` polls, and frees a slot only when the+/// oldest poll is covered.+pub(crate) struct CappedPolling {+ collector: Collector,+ settlement: Arc<dyn Settlement>,+ max_in_flight_polls: usize,+ in_flight_polls: VecDeque<InFlightPoll>,+ accepting_new_batches: bool,+ commit_due: bool,+ group_id: String,+ debug_recorder: Option<Arc<DebugRecorder>>,+}++impl CappedPolling {+ pub(crate) fn new(parts: Parts, max_in_flight_polls: usize) -> Self {+ Self {+ collector: Collector::new(parts.limits),+ settlement: parts.settlement,+ max_in_flight_polls: max_in_flight_polls.max(1),+ in_flight_polls: VecDeque::new(),+ accepting_new_batches: true,+ commit_due: false,+ group_id: parts.group_id,+ debug_recorder: parts.debug_recorder,+ }+ }+}++impl PollingStrategy for CappedPolling {+ fn reading(&mut self) -> bool {+ self.accepting_new_batches && self.in_flight_polls.len() < self.max_in_flight_polls+ }++ fn deadline(&self) -> Option<Instant> {+ self.collector.deadline()+ }++ fn received<M: Message>(&mut self, message: &M, now: Instant) {+ self.collector.add(message, now);+ }++ fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {+ let settlement = &*self.settlement;+ apply_completion(+ &mut self.in_flight_polls,+ completion,+ |tp, generation, run| settlement.settle_run(tp, generation, run).map(drop),+ )?;+ self.commit_due = true;+ Ok(())+ }++ fn ticked(&mut self) {+ self.commit_due = true;+ }++ fn revoked(&mut self, topic_partitions: &[TopicPartition]) {+ let building = self.collector.remove_partitions(topic_partitions) as u64;+ let in_flight = strip_revoked_partitions(&mut self.in_flight_polls, topic_partitions);+ self.report_in_flight();+ report_revoked(building + in_flight);+ }++ fn shutdown(&mut self) {+ self.accepting_new_batches = false;+ self.collector.discard();+ }++ fn submitted(&mut self, poll: InFlightPoll) {+ self.in_flight_polls.push_back(poll);+ self.report_in_flight();+ }++ fn next_step(&mut self, now: Instant) -> Option<Step> {+ while self+ .in_flight_polls+ .front()+ .is_some_and(InFlightPoll::is_complete)+ {+ if let Err(err) = self.retire_oldest_poll() {+ return Some(Step::Fail(err));+ }+ self.commit_due = true;+ }+ if std::mem::take(&mut self.commit_due) {+ if let Some(commit) = self.settlement.take_due(now.into_std()) {+ return Some(Step::Commit(commit));+ }+ }+ if let Some(collected) = self.collector.take_ready(now) {+ return Some(Step::Submit(collected));+ }+ (!self.accepting_new_batches && self.in_flight_polls.is_empty()).then_some(Step::Stop)+ }+}++impl CappedPolling {+ fn retire_oldest_poll(&mut self) -> anyhow::Result<()> {+ let poll = pop_committable(&mut self.in_flight_polls)?;+ self.report_in_flight();+ self.settlement.settle_poll(&poll.charged_slices())?;+ emit_latest_processed_timestamp_metrics(&poll.partitions, &self.group_id);+ record_if(&self.debug_recorder, || DebugEventKind::PollCompleted {+ batch_id: poll.poll_id.clone(),+ accepted: poll.accepted,+ duration_ms: poll.dispatched_at.elapsed().as_millis() as u64,+ partitions: debug_partition_offsets(&poll.partitions),+ });+ counter!("ingestion_consumer_messages_processed_total").increment(poll.accepted as u64);+ counter!("ingestion_consumer_batches_processed_total").increment(1);+ Ok(())+ }++ fn report_in_flight(&self) {+ gauge!("ingestion_consumer_in_flight_batches").set(self.in_flight_polls.len() as f64);+ }+}++/// Keeps the other partitions' slices: a dropped slice never settles, and+/// its partition's commits stop at that hole.+fn strip_revoked_partitions(+ in_flight_polls: &mut VecDeque<InFlightPoll>,+ revoked: &[TopicPartition],+) -> u64 {+ let mut stripped: u64 = 0;+ for poll in in_flight_polls.iter_mut() {+ let mut removed_delivered = 0u32;+ let mut removed_covered = 0u32;+ let mut removed_accepted = 0u32;+ poll.partitions.retain(|topic_partition, deliveries| {+ if revoked.contains(topic_partition) {+ removed_delivered += deliveries.delivered;+ removed_covered += deliveries.covered;+ removed_accepted += deliveries.accepted;+ false+ } else {+ true+ }+ });+ if removed_delivered > 0 {+ stripped += u64::from(removed_delivered);+ poll.message_count = poll.message_count.saturating_sub(removed_delivered);+ poll.covered = poll.covered.saturating_sub(removed_covered);+ poll.accepted = poll.accepted.saturating_sub(removed_accepted);+ }+ }+ in_flight_polls.retain(|poll| poll.message_count > 0);+ stripped+}++/// A key-table run can merge several polls into one send, so a completion+/// can span polls. Within one epoch, spans are disjoint per partition, so an+/// offset matches at most one poll.+fn apply_completion(+ in_flight: &mut VecDeque<InFlightPoll>,+ completion: &GroupCompletion,+ mut settle: impl FnMut(&TopicPartition, u64, &[Offset]) -> anyhow::Result<()>,+) -> anyhow::Result<()> {+ let fully_accepted = completion.accepted as usize == completion.offsets.len();+ let mut accepted = completion.accepted;+ let mut unmatched: u64 = 0;+ let mut remaining = completion.offsets.as_slice();+ 'runs: while let Some(first) = remaining.first() {+ for poll in in_flight+ .iter_mut()+ .filter(|poll| poll.assignment_epoch == completion.assignment_epoch)+ {+ for (topic_partition, deliveries) in poll.partitions.iter_mut() {+ let span = deliveries.span;+ if topic_partition.partition != completion.partition.0+ || first.0 < span.first+ || span.last < first.0+ {+ continue;+ }+ let run_len = remaining+ .iter()+ .take_while(|offset| span.first <= offset.0 && offset.0 <= span.last)+ .count();+ let (run, rest) = remaining.split_at(run_len);+ let run_accepted = accepted.min(run_len as u32);+ accepted -= run_accepted;+ poll.covered += run_len as u32;+ poll.accepted += run_accepted;+ deliveries.covered += run_len as u32;+ deliveries.accepted += run_accepted;+ let settleable = deliveries.credit_run(run);+ if fully_accepted && !settleable.is_empty() {+ settle(topic_partition, deliveries.generation, &settleable)?;+ }+ remaining = rest;+ continue 'runs;+ }+ }+ unmatched += 1;+ remaining = &remaining[1..];+ }+ if unmatched > 0 {+ counter!("ingestion_consumer_stale_group_completions_total").increment(1);+ warn!(+ partition = %completion.partition,+ unmatched,+ epoch = completion.assignment_epoch,+ "Discarding completion offsets that match no in-flight poll"+ );+ }+ Ok(())+}++fn pop_committable(in_flight: &mut VecDeque<InFlightPoll>) -> anyhow::Result<InFlightPoll> {+ let poll = in_flight.pop_front().expect("front is present");+ if poll.accepted < poll.message_count {+ anyhow::bail!(+ "accepted {}/{} messages — not committing offsets",+ poll.accepted,+ poll.message_count+ );+ }+ if let Some((topic_partition, offset)) = poll.uncredited_charge() {+ counter!("ingestion_consumer_commit_violations_total", "kind" => "uncredited").increment(1);+ anyhow::bail!(+ "offset {} of {}/{} has no completion — not committing offsets",+ offset.0,+ topic_partition.topic,+ topic_partition.partition+ );+ }+ Ok(poll)+}++fn emit_latest_processed_timestamp_metrics(+ partitions: &HashMap<TopicPartition, PartitionDeliveries>,+ group_id: &str,+) {+ // Same name as the Node.js consumer's gauge, so one dashboard reads both.+ for (topic_partition, partition) in partitions {+ gauge!(+ "latest_processed_timestamp_ms",+ "topic" => topic_partition.topic.clone(),+ "partition" => topic_partition.partition.to_string(),+ "groupId" => group_id.to_string()+ )+ .set(partition.latest_kafka_ts as f64);+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashSet;+ use std::time::{Duration, Instant};++ use common_kafka_consumer::{Charge, Offset as MessageOffset, Partition, TopicOffsetLedger};++ use super::super::harness::{p, Harness};+ use super::*;+ use crate::order_sentinel::OffsetSpan;+ use crate::partition_budget::BudgetLimits;+ use crate::settlement::{charge_offsets, GroupSettlement};++ fn poll(epoch: u64, partition: i32, first: i64, last: i64, count: u32) -> InFlightPoll {+ let mut partitions = HashMap::new();+ partitions.insert(+ TopicPartition::new("test", partition),+ PartitionDeliveries {+ span: OffsetSpan { first, last },+ generation: 0,+ payload_bytes: 0,+ charges: (first..=last)+ .map(|offset| (MessageOffset(offset), Charge::ZERO))+ .collect(),+ latest_kafka_ts: 0,+ max_lag_ms: None,+ delivered: count,+ covered: 0,+ accepted: 0,+ credited: HashSet::default(),+ },+ );+ InFlightPoll {+ poll_id: format!("poll-{epoch}-{partition}-{first}"),+ assignment_epoch: epoch,+ partitions,+ uncommitted: HashMap::new(),+ message_count: count,+ covered: 0,+ accepted: 0,+ dispatched_at: Instant::now(),+ }+ }++ fn completion(epoch: u64, partition: i32, offsets: &[i64], accepted: u32) -> GroupCompletion {+ GroupCompletion {+ partition: Partition(partition),+ assignment_epoch: epoch,+ generation: 0,+ offsets: offsets.iter().map(|o| MessageOffset(*o)).collect(),+ accepted,+ }+ }++ fn credit(in_flight: &mut VecDeque<InFlightPoll>, completion: GroupCompletion) {+ apply_completion(in_flight, &completion, |_, _, _| Ok(())).unwrap();+ }++ #[test]+ fn an_accepted_completion_settles_one_run_per_poll() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);+ in_flight[1]+ .partitions+ .get_mut(&TopicPartition::new("test", 0))+ .unwrap()+ .generation = 1;+ let mut runs = Vec::new();++ apply_completion(+ &mut in_flight,+ &completion(1, 0, &[2, 3, 4, 5], 4),+ |topic_partition, generation, offsets| {+ runs.push((topic_partition.partition, generation, offsets.to_vec()));+ Ok(())+ },+ )+ .unwrap();++ assert_eq!(+ runs,+ vec![+ (0, 0, vec![MessageOffset(2), MessageOffset(3)]),+ (0, 1, vec![MessageOffset(4), MessageOffset(5)]),+ ],+ "each run carries its own poll's generation"+ );+ }++ #[test]+ fn a_completion_spanning_two_polls_commits_once_the_prefix_is_whole() {+ let ledger = Arc::new(TopicOffsetLedger::new());+ let settlement =+ GroupSettlement::new(Arc::clone(&ledger), Duration::ZERO, BudgetLimits::unset());+ let partition = TopicPartition::new("test", 0);+ charge_offsets(&ledger, &partition, 0, &[0, 1, 2, 3, 4, 5, 6, 7]);+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);+ let mut settle = |topic_partition: &TopicPartition, generation: u64, offsets: &[Offset]| {+ settlement+ .settle_run(topic_partition, generation, offsets)+ .map(drop)+ };++ apply_completion(+ &mut in_flight,+ &completion(1, 0, &[2, 3, 4, 5], 4),+ &mut settle,+ )+ .unwrap();+ assert!(+ settlement.take_due(Instant::now()).is_none(),+ "offsets 0 and 1 are still in flight"+ );++ apply_completion(+ &mut in_flight,+ &completion(1, 0, &[0, 1, 6, 7], 4),+ &mut settle,+ )+ .unwrap();+ assert_eq!(+ settlement+ .take_due(Instant::now())+ .map(|commit| commit.offsets),+ Some(HashMap::from([(partition, MessageOffset(8))]))+ );+ }++ #[test]+ fn a_short_accepted_completion_settles_nothing() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4)]);+ let mut runs = 0;++ apply_completion(&mut in_flight, &completion(1, 0, &[0, 1], 1), |_, _, _| {+ runs += 1;+ Ok(())+ })+ .unwrap();++ assert_eq!(runs, 0, "the ledger must not commit past a dropped message");+ assert_eq!(in_flight[0].covered, 2);+ assert_eq!(in_flight[0].accepted, 1);+ }++ #[test]+ fn apply_completion_spanning_two_polls_credits_each() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);++ credit(&mut in_flight, completion(1, 0, &[2, 3, 4, 5], 4));++ assert_eq!(in_flight[0].covered, 2);+ assert_eq!(in_flight[0].accepted, 2);+ assert_eq!(in_flight[1].covered, 2);+ assert_eq!(in_flight[1].accepted, 2);+ }++ #[test]+ fn apply_completion_under_report_shorts_the_tail_poll() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 0, 4, 7, 4)]);++ credit(&mut in_flight, completion(1, 0, &[2, 3, 4, 5], 3));++ assert_eq!(in_flight[0].accepted, 2, "leading offsets are accepted");+ assert_eq!(+ in_flight[1].accepted, 1,+ "the tail poll fails its accepted check"+ );+ assert_eq!(in_flight[1].covered, 2);+ }++ #[test]+ fn a_complete_poll_with_a_charged_offset_no_completion_covered_is_not_committable() {+ let mut in_flight = VecDeque::from([poll(1, 0, 10, 12, 3)]);++ credit(&mut in_flight, completion(1, 0, &[10, 12, 12], 3));++ assert!(in_flight[0].is_complete());+ let error = pop_committable(&mut in_flight)+ .err()+ .expect("offset 11 has no completion");+ assert!(error.to_string().contains("offset 11 "), "{error}");+ }++ #[test]+ fn a_poll_whose_deliveries_span_an_offset_gap_commits() {+ let mut gapped = poll(1, 0, 0, 0, 2);+ let deliveries = gapped+ .partitions+ .get_mut(&TopicPartition::new("test", 0))+ .unwrap();+ deliveries.span.last = i64::MAX;+ deliveries+ .charges+ .push((MessageOffset(i64::MAX), Charge::ZERO));+ let mut in_flight = VecDeque::from([gapped]);++ credit(&mut in_flight, completion(1, 0, &[0, i64::MAX], 2));++ assert!(in_flight[0].is_complete());+ assert!(pop_committable(&mut in_flight).is_ok());+ }++ #[test]+ fn strip_revoked_partitions_keeps_the_other_partitions_slices() {+ let mut two = poll(1, 0, 0, 1, 4);+ two.partitions.insert(+ TopicPartition::new("test", 1),+ PartitionDeliveries {+ span: OffsetSpan {+ first: 10,+ last: 11,+ },+ generation: 0,+ payload_bytes: 0,+ charges: Vec::new(),+ latest_kafka_ts: 0,+ max_lag_ms: None,+ delivered: 2,+ covered: 0,+ accepted: 0,+ credited: HashSet::default(),+ },+ );+ two.partitions+ .get_mut(&TopicPartition::new("test", 0))+ .unwrap()+ .delivered = 2;+ let mut in_flight = VecDeque::from([two]);+ credit(&mut in_flight, completion(1, 1, &[10], 1));++ let stripped = strip_revoked_partitions(&mut in_flight, &[TopicPartition::new("test", 1)]);++ assert_eq!(stripped, 2);+ assert_eq!(in_flight[0].message_count, 2);+ assert_eq!(in_flight[0].covered, 0, "the revoked slice's credit goes");+ assert!(!in_flight[0].is_complete());++ credit(&mut in_flight, completion(1, 1, &[11], 1));+ assert_eq!(in_flight[0].covered, 0);+ credit(&mut in_flight, completion(1, 0, &[0, 1], 2));+ assert!(in_flight[0].is_complete());+ assert_eq!(in_flight[0].accepted, 2);+ assert!(pop_committable(&mut in_flight).is_ok());+ }++ #[test]+ fn strip_revoked_partitions_removes_an_emptied_poll() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(1, 2, 0, 3, 4)]);++ let stripped = strip_revoked_partitions(&mut in_flight, &[TopicPartition::new("test", 0)]);++ assert_eq!(stripped, 4);+ assert_eq!(in_flight.len(), 1);+ assert_eq!(in_flight[0].message_count, 4);+ }++ #[test]+ fn apply_completion_requires_a_matching_epoch() {+ let mut in_flight = VecDeque::from([poll(1, 0, 0, 3, 4), poll(2, 0, 0, 3, 4)]);++ credit(&mut in_flight, completion(2, 0, &[0, 1, 2, 3], 4));++ assert_eq!(in_flight[0].covered, 0);+ assert_eq!(in_flight[1].covered, 4);+ }++ fn capped(max_in_flight_polls: usize) -> Harness<CappedPolling> {+ Harness::new(|parts| CappedPolling::new(parts, max_in_flight_polls))+ }++ #[test]+ fn the_capped_strategy_stops_reading_at_its_cap_until_the_oldest_poll_is_covered() {+ let mut harness = capped(1);+ assert!(harness.poller.reading());++ harness.receive(0, &[0, 1]);+ assert!(harness.step().is_none(), "the poll is still collecting");+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(!harness.poller.reading());++ harness.complete(0, &[0, 1]);+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));+ assert!(harness.poller.reading());+ }+}diff --git a/rust/ingestion-consumer/src/polling/collector.rs b/rust/ingestion-consumer/src/polling/collector.rsnew file mode 100644index 000000000000..1067b9cce5d2--- /dev/null+++ b/rust/ingestion-consumer/src/polling/collector.rs@@ -0,0 +1,604 @@+use std::collections::HashMap;+use std::sync::Arc;+use std::time::Duration;++use common_kafka_consumer::{Charge, Partition, TopicPartition};+use metrics::counter;+use rdkafka::message::{Headers, Message};+use tokio::time::Instant;++use super::poll::{BatchStats, CollectedBatch, Delivery, PartitionDeliveries};+use crate::types::{Accumulator, SerializedKafkaMessage};++#[derive(Debug, Clone, Copy)]+pub(crate) struct PollLimits {+ pub(crate) batch_size: usize,+ /// Payload-byte bound; `0` disables it.+ pub(crate) batch_size_bytes: usize,+ pub(crate) batch_timeout: Duration,+}++pub(crate) struct Collector {+ limits: PollLimits,+ deadline: Option<Instant>,+ accumulator: Accumulator,+ partitions: HashMap<TopicPartition, PartitionDeliveries>,+ stats: BatchStats,+ started_ms: i64,+ current_topic: Option<Arc<str>>,+}++impl Collector {+ pub(crate) fn new(limits: PollLimits) -> Self {+ Self {+ limits,+ deadline: None,+ accumulator: Accumulator::default(),+ partitions: HashMap::new(),+ stats: BatchStats::default(),+ started_ms: 0,+ current_topic: None,+ }+ }++ /// The first message of a poll starts its batch timeout.+ pub(crate) fn add(&mut self, message: &impl Message, now: Instant) {+ if self.deadline.is_none() {+ self.deadline = Some(now + self.limits.batch_timeout);+ self.started_ms = current_time_ms();+ }+ if self.current_topic.as_deref() != Some(message.topic()) {+ self.current_topic = Some(Arc::from(message.topic()));+ }+ let topic = Arc::clone(self.current_topic.as_ref().expect("set above"));+ let partition = message.partition();+ let offset = message.offset();+ let kafka_ts = message.timestamp().to_millis().unwrap_or(0);++ let payload_bytes = message.payload().map(|v| v.len()).unwrap_or(0);+ self.stats.total_bytes += payload_bytes;++ let mut headers = HashMap::new();+ if let Some(rdkafka_headers) = message.headers() {+ for i in 0..rdkafka_headers.count() {+ let header = rdkafka_headers.get(i);+ if let Some(value) = header.value {+ if let Ok(value_str) = std::str::from_utf8(value) {+ headers.insert(header.key.to_string(), value_str.to_string());+ }+ }+ }+ }++ let lag_ms = headers+ .get("now")+ .and_then(|v| parse_now_ms(v))+ .map(|capture_ms| (self.started_ms - capture_ms).max(0));+ if let Some(lag_ms) = lag_ms {+ self.stats.message_lags_ms.push((partition, lag_ms));+ }++ let delivery = Delivery {+ offset,+ charge: message_charge(message),+ payload_bytes,+ kafka_ts,+ lag_ms,+ };+ let key = TopicPartition::new(topic.to_string(), partition);+ match self.partitions.get_mut(&key) {+ Some(deliveries) => deliveries.record(&delivery),+ None => {+ self.partitions+ .insert(key, PartitionDeliveries::new(&delivery));+ }+ }++ let serialized = SerializedKafkaMessage {+ topic,+ partition,+ offset,+ timestamp: kafka_ts,+ key: message+ .key()+ .and_then(|k| std::str::from_utf8(k).ok())+ .map(|s| s.to_string()),+ value: message+ .payload()+ .and_then(|v| std::str::from_utf8(v).ok())+ .map(|s| s.to_string()),+ headers,+ generation: 0,+ };++ self.accumulator+ .push(Partition(partition), serialized.into());+ }++ pub(crate) fn deadline(&self) -> Option<Instant> {+ self.deadline+ }++ pub(crate) fn take_ready(&mut self, now: Instant) -> Option<CollectedBatch> {+ let deadline = self.deadline?;+ if deadline > now && !self.is_full() {+ return None;+ }+ self.deadline = None;+ Some(CollectedBatch {+ accumulator: std::mem::take(&mut self.accumulator),+ partitions: std::mem::take(&mut self.partitions),+ stats: std::mem::take(&mut self.stats),+ })+ }++ /// Returns how many messages it removed.+ pub(crate) fn remove_partitions(&mut self, topic_partitions: &[TopicPartition]) -> usize {+ let mut removed = 0;+ for topic_partition in topic_partitions {+ let Some(deliveries) = self.partitions.remove(topic_partition) else {+ continue;+ };+ let partition = topic_partition.partition;+ self.accumulator.remove_partition(Partition(partition));+ self.stats.total_bytes -= deliveries.payload_bytes;+ self.stats+ .message_lags_ms+ .retain(|(lagging, _)| *lagging != partition);+ removed += deliveries.delivered as usize;+ }+ if self.partitions.is_empty() {+ self.discard();+ }+ removed+ }++ pub(crate) fn discard(&mut self) {+ self.deadline = None;+ self.accumulator = Accumulator::default();+ self.partitions.clear();+ self.stats = BatchStats::default();+ }++ /// The byte bound counts only messages already added, so a poll always+ /// carries one message, even one larger than the bound. The overshoot is+ /// at most one message, which the topic's `max.message.bytes` bounds.+ fn is_full(&self) -> bool {+ if self.accumulator.message_count() >= self.limits.batch_size {+ return true;+ }+ if self.limits.batch_size_bytes > 0+ && self.stats.total_bytes >= self.limits.batch_size_bytes+ {+ counter!("ingestion_consumer_batches_byte_capped_total").increment(1);+ return true;+ }+ false+ }+}++/// The byte count bounds memory, so it counts the key and every header key+/// and value, not only the payload.+fn message_charge(message: &impl Message) -> Charge {+ let payload_bytes = message.payload().map(|v| v.len()).unwrap_or(0);+ let key_bytes = message.key().map(|k| k.len()).unwrap_or(0);+ let mut header_bytes = 0;+ if let Some(headers) = message.headers() {+ for i in 0..headers.count() {+ let header = headers.get(i);+ header_bytes += header.key.len() + header.value.map(|v| v.len()).unwrap_or(0);+ }+ }+ Charge {+ events: 1,+ bytes: (payload_bytes + key_bytes + header_bytes) as u64,+ }+}++fn current_time_ms() -> i64 {+ std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default()+ .as_millis() as i64+}++fn parse_now_ms(value: &str) -> Option<i64> {+ chrono::DateTime::parse_from_rfc3339(value)+ .ok()+ .map(|dt| dt.timestamp_millis())+}++#[cfg(test)]+pub(crate) mod tests {+ use common_kafka_consumer::Offset;+ use metrics_util::debugging::{DebugValue, DebuggingRecorder};+ use rdkafka::message::{Header, OwnedHeaders, OwnedMessage, Timestamp};++ use super::*;++ const BATCH_TIMEOUT: Duration = Duration::from_millis(500);++ struct Sent {+ partition: i32,+ offset: i64,+ key: Option<Vec<u8>>,+ payload: Option<Vec<u8>>,+ timestamp_ms: i64,+ headers: Vec<(&'static str, Vec<u8>)>,+ }++ impl Sent {+ fn new(partition: i32, offset: i64) -> Self {+ Self {+ partition,+ offset,+ key: Some(b"tok:user-1".to_vec()),+ payload: Some(b"{}".to_vec()),+ timestamp_ms: 1_000,+ headers: Vec::new(),+ }+ }++ fn message(&self) -> OwnedMessage {+ let headers = self+ .headers+ .iter()+ .fold(OwnedHeaders::new(), |headers, (key, value)| {+ headers.insert(Header {+ key,+ value: Some(value),+ })+ });+ OwnedMessage::new(+ self.payload.clone(),+ self.key.clone(),+ "events".to_string(),+ Timestamp::CreateTime(self.timestamp_ms),+ self.partition,+ self.offset,+ Some(headers),+ )+ }+ }++ pub(crate) fn message(partition: i32, offset: i64, payload_bytes: usize) -> OwnedMessage {+ Sent {+ key: Some(format!("tok:user-{offset}").into_bytes()),+ payload: Some(vec![b'x'; payload_bytes]),+ ..Sent::new(partition, offset)+ }+ .message()+ }++ fn collector(batch_size: usize, batch_size_bytes: usize) -> Collector {+ Collector::new(PollLimits {+ batch_size,+ batch_size_bytes,+ batch_timeout: BATCH_TIMEOUT,+ })+ }++ fn collect(sent: &[Sent]) -> CollectedBatch {+ let mut collector = collector(100, 0);+ let now = Instant::now();+ for sent in sent {+ collector.add(&sent.message(), now);+ }+ collector+ .take_ready(now + BATCH_TIMEOUT)+ .expect("the batch timeout passed")+ }++ fn captured(seconds_ago: i64) -> Vec<u8> {+ (chrono::Utc::now() - chrono::Duration::seconds(seconds_ago))+ .to_rfc3339()+ .into_bytes()+ }++ fn deliveries(batch: &CollectedBatch, partition: i32) -> &PartitionDeliveries {+ &batch.partitions[&TopicPartition::new("events", partition)]+ }++ fn offsets(batch: &CollectedBatch, partition: i32) -> Option<(i64, i64)> {+ batch+ .partitions+ .get(&TopicPartition::new("events", partition))+ .map(|deliveries| (deliveries.span.first, deliveries.span.last))+ }++ fn groups(batch: CollectedBatch) -> Vec<(i32, Option<String>, Vec<i64>)> {+ batch+ .accumulator+ .into_groups()+ .into_iter()+ .map(|group| {+ let offsets = group.messages.iter().map(|m| m.offset.0).collect();+ (group.partition.0, group.key, offsets)+ })+ .collect()+ }++ #[test]+ fn a_poll_carries_each_message_whole() {+ let batch = collect(&[Sent {+ key: Some(b"tok:user-1".to_vec()),+ payload: Some(br#"{"event":"$pageview"}"#.to_vec()),+ timestamp_ms: 1_234,+ headers: vec![("token", b"tok".to_vec()), ("binary", vec![0xff, 0xfe])],+ ..Sent::new(3, 42)+ }]);++ let mut groups = batch.accumulator.into_groups();+ assert_eq!(groups.len(), 1);+ let group = groups.remove(0);+ assert_eq!(group.partition, Partition(3));+ assert_eq!(group.key.as_deref(), Some("tok:user-1"));+ assert_eq!(group.messages.len(), 1);+ assert_eq!(group.messages[0].offset, Offset(42));+ let message = &group.messages[0].message;+ assert_eq!(&*message.topic, "events");+ assert_eq!(message.partition, 3);+ assert_eq!(message.offset, 42);+ assert_eq!(message.timestamp, 1_234);+ assert_eq!(message.key.as_deref(), Some("tok:user-1"));+ assert_eq!(message.value.as_deref(), Some(r#"{"event":"$pageview"}"#));+ assert_eq!(+ message.headers,+ HashMap::from([("token".to_string(), "tok".to_string())]),+ "a header that is not UTF-8 is left out"+ );+ }++ #[test]+ fn a_message_that_is_not_utf8_keeps_its_place_without_its_text() {+ let batch = collect(&[Sent {+ key: Some(vec![0xff]),+ payload: Some(vec![0xff, 0xfe, 0xfd]),+ ..Sent::new(0, 7)+ }]);++ assert_eq!(batch.stats.total_bytes, 3);+ assert_eq!(deliveries(&batch, 0).delivered, 1);+ let group = batch.accumulator.into_groups().remove(0);+ assert_eq!(group.key, None, "a key that is not UTF-8 routes as keyless");+ assert_eq!(group.messages[0].offset, Offset(7));+ assert_eq!(group.messages[0].message.value, None);+ }++ #[test]+ fn a_poll_groups_messages_per_partition_and_key_in_the_order_received() {+ let keyed = |partition, offset, key: &str| Sent {+ key: Some(key.as_bytes().to_vec()),+ ..Sent::new(partition, offset)+ };+ let batch = collect(&[+ keyed(0, 0, "a"),+ keyed(0, 1, "b"),+ keyed(1, 5, "a"),+ keyed(0, 2, "a"),+ Sent {+ key: None,+ ..Sent::new(0, 3)+ },+ ]);++ assert_eq!(+ groups(batch),+ vec![+ (0, Some("a".to_string()), vec![0, 2]),+ (0, Some("b".to_string()), vec![1]),+ (1, Some("a".to_string()), vec![5]),+ (0, None, vec![3]),+ ]+ );+ }++ #[test]+ fn a_poll_records_each_partitions_span_charges_and_latest_timestamp() {+ let sent = |offset, timestamp_ms| Sent {+ key: Some(b"k".to_vec()),+ payload: Some(vec![b'x'; 10]),+ timestamp_ms,+ headers: vec![("h", b"vv".to_vec())],+ ..Sent::new(0, offset)+ };+ let batch = collect(&[sent(10, 5_000), sent(12, 3_000), Sent::new(1, 20)]);++ let p0 = deliveries(&batch, 0);+ assert_eq!((p0.span.first, p0.span.last), (10, 12));+ assert_eq!(p0.delivered, 2);+ let each = Charge {+ events: 1,+ bytes: 10 + 1 + 3,+ };+ assert_eq!(+ p0.charges,+ vec![(Offset(10), each), (Offset(12), each)],+ "a charge counts the payload, the key, and every header"+ );+ assert_eq!(p0.latest_kafka_ts, 5_000);+ assert_eq!(deliveries(&batch, 1).delivered, 1);+ }++ #[test]+ fn a_poll_measures_lag_from_each_messages_now_header() {+ let with_now = |partition, offset, now: Vec<u8>| Sent {+ headers: vec![("now", now)],+ ..Sent::new(partition, offset)+ };+ let batch = collect(&[+ with_now(0, 0, captured(60)),+ with_now(0, 1, captured(10)),+ with_now(1, 0, b"not a timestamp".to_vec()),+ Sent::new(1, 1),+ ]);++ let lags: Vec<(i32, i64)> = batch.stats.message_lags_ms.clone();+ assert_eq!(lags.len(), 2, "{lags:?}");+ assert!(lags.iter().all(|(partition, _)| *partition == 0));+ assert!((60_000..61_000).contains(&lags[0].1), "{lags:?}");+ assert!((10_000..11_000).contains(&lags[1].1), "{lags:?}");+ assert_eq!(deliveries(&batch, 0).max_lag_ms, Some(lags[0].1));+ assert_eq!(deliveries(&batch, 1).max_lag_ms, None);+ assert_eq!(batch.stats.total_bytes, 8, "four two-byte payloads");+ }++ struct Bound {+ batch_size: usize,+ batch_size_bytes: usize,+ payloads: Vec<usize>,+ expected_ready_after: usize,+ expected_byte_capped: u64,+ }++ #[rstest::rstest]+ #[case::the_count_bound(Bound { batch_size: 3, batch_size_bytes: 0, payloads: vec![10; 5], expected_ready_after: 3, expected_byte_capped: 0 })]+ #[case::the_byte_bound(Bound { batch_size: 100, batch_size_bytes: 25, payloads: vec![10; 5], expected_ready_after: 3, expected_byte_capped: 1 })]+ #[case::an_oversized_first_message(Bound { batch_size: 100, batch_size_bytes: 25, payloads: vec![100, 10], expected_ready_after: 1, expected_byte_capped: 1 })]+ #[case::no_byte_bound(Bound { batch_size: 4, batch_size_bytes: 0, payloads: vec![1_000; 5], expected_ready_after: 4, expected_byte_capped: 0 })]+ fn a_poll_is_ready_at_its_first_bound(#[case] bound: Bound) {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);+ let mut collector = collector(bound.batch_size, bound.batch_size_bytes);+ let now = Instant::now();++ let mut ready_after = None;+ for (offset, &payload) in bound.payloads.iter().enumerate() {+ collector.add(&message(0, offset as i64, payload), now);+ if collector.take_ready(now).is_some() {+ ready_after = Some(offset + 1);+ break;+ }+ }++ assert_eq!(ready_after, Some(bound.expected_ready_after));+ let byte_capped = snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .find_map(|(key, _, _, value)| match value {+ DebugValue::Counter(count)+ if key.key().name() == "ingestion_consumer_batches_byte_capped_total" =>+ {+ Some(count)+ }+ _ => None,+ })+ .unwrap_or(0);+ assert_eq!(byte_capped, bound.expected_byte_capped);+ }++ #[test]+ fn a_poll_is_ready_once_its_first_message_is_a_batch_timeout_old() {+ let mut collector = collector(100, 0);+ let first = Instant::now();+ assert_eq!(+ collector.deadline(),+ None,+ "an empty collector has no deadline"+ );++ collector.add(&message(0, 0, 1), first);+ collector.add(&message(0, 1, 1), first + Duration::from_millis(400));++ assert_eq!(collector.deadline(), Some(first + BATCH_TIMEOUT));+ assert!(collector+ .take_ready(first + Duration::from_millis(499))+ .is_none());+ let batch = collector+ .take_ready(first + BATCH_TIMEOUT)+ .expect("the timeout passed");+ assert_eq!(offsets(&batch, 0), Some((0, 1)));+ assert_eq!(collector.deadline(), None);+ }++ #[test]+ fn the_next_poll_starts_empty_with_its_own_timeout() {+ let mut collector = collector(100, 0);+ let first = Instant::now();+ collector.add(&message(0, 0, 5), first);+ collector.take_ready(first + BATCH_TIMEOUT);++ let second = first + Duration::from_secs(2);+ collector.add(&message(1, 9, 7), second);++ assert_eq!(collector.deadline(), Some(second + BATCH_TIMEOUT));+ let batch = collector+ .take_ready(second + BATCH_TIMEOUT)+ .expect("the second timeout passed");+ assert_eq!(batch.accumulator.message_count(), 1);+ assert_eq!(offsets(&batch, 0), None);+ assert_eq!(offsets(&batch, 1), Some((9, 9)));+ assert_eq!(batch.stats.total_bytes, 7);+ }++ #[test]+ fn a_removed_partition_leaves_the_poll() {+ let mut collector = collector(100, 0);+ let now = Instant::now();+ let with_now = |partition, offset| Sent {+ payload: Some(vec![b'x'; 5]),+ headers: vec![("now", captured(1))],+ ..Sent::new(partition, offset)+ };+ collector.add(&with_now(0, 10).message(), now);+ collector.add(&with_now(1, 20).message(), now);+ collector.add(&with_now(0, 11).message(), now);++ let removed = collector.remove_partitions(&[TopicPartition::new("events", 0)]);+ collector.add(&message(0, 10, 7), now);++ assert_eq!(removed, 2);+ let batch = collector+ .take_ready(now + BATCH_TIMEOUT)+ .expect("partition 1 and the redelivery stay");+ assert_eq!(batch.stats.total_bytes, 12);+ assert_eq!(+ batch+ .stats+ .message_lags_ms+ .iter()+ .map(|(partition, _)| *partition)+ .collect::<Vec<_>>(),+ vec![1]+ );+ assert_eq!(+ groups(batch),+ vec![+ (1, Some("tok:user-1".to_string()), vec![20]),+ (0, Some("tok:user-10".to_string()), vec![10]),+ ]+ );+ }++ #[test]+ fn removing_every_partition_ends_the_poll() {+ let mut collector = collector(100, 0);+ collector.add(&message(0, 10, 5), Instant::now());++ collector.remove_partitions(&[TopicPartition::new("events", 0)]);++ assert_eq!(collector.deadline(), None);+ assert!(collector+ .take_ready(Instant::now() + Duration::from_secs(1))+ .is_none());+ }++ #[test]+ fn a_discarded_poll_leaves_nothing_for_the_next() {+ let mut collector = collector(100, 0);+ let now = Instant::now();+ collector.add(&message(0, 10, 5), now);++ collector.discard();+ assert_eq!(collector.deadline(), None);+ collector.add(&message(0, 11, 5), now);++ let batch = collector+ .take_ready(now + BATCH_TIMEOUT)+ .expect("the timeout passed");+ assert_eq!(offsets(&batch, 0), Some((11, 11)));+ assert_eq!(batch.stats.total_bytes, 5);+ }+}diff --git a/rust/ingestion-consumer/src/polling/continuous.rs b/rust/ingestion-consumer/src/polling/continuous.rsnew file mode 100644index 000000000000..9358598ab6fa--- /dev/null+++ b/rust/ingestion-consumer/src/polling/continuous.rs@@ -0,0 +1,470 @@+use std::collections::{HashMap, VecDeque};+use std::sync::Arc;++use common_kafka_consumer::{Charge, GroupCompletion, Partition, TopicPartition};+use metrics::{counter, gauge};+use rdkafka::message::Message;+use tokio::time::Instant;++use super::poll::InFlightPoll;+use super::{report_revoked, Collector, Parts, PollingStrategy, Step};+use crate::charge_limit::ChargeLimit;+use crate::settlement::{Commit, RunOutcome, Settlement};++/// Nothing waits on a submitted poll: the partition budget and the in-flight+/// limit bound admission.+pub(crate) struct ContinuousPolling {+ collector: Collector,+ settlement: Arc<dyn Settlement>,+ uncommitted: UncommittedCharges,+ processed: ProcessedTimestamps,+ limits: ChargeLimit,+ at_limit: bool,+ accepting_new_batches: bool,+ commit_due: bool,+}++impl ContinuousPolling {+ pub(crate) fn new(parts: Parts, limits: ChargeLimit) -> Self {+ Self {+ collector: Collector::new(parts.limits),+ settlement: parts.settlement,+ uncommitted: UncommittedCharges::default(),+ processed: ProcessedTimestamps::new(parts.group_id),+ limits,+ at_limit: false,+ accepting_new_batches: true,+ commit_due: false,+ }+ }+}++impl PollingStrategy for ContinuousPolling {+ /// The poll in progress is not counted, so the uncommitted charge can+ /// pass the limit by one poll.+ fn reading(&mut self) -> bool {+ let at_limit = self.limits.reached(self.uncommitted.total());+ if at_limit && !self.at_limit {+ counter!("ingestion_consumer_in_flight_limit_reached_total").increment(1);+ }+ self.at_limit = at_limit;+ self.accepting_new_batches && !at_limit+ }++ fn deadline(&self) -> Option<Instant> {+ self.collector.deadline()+ }++ fn received<M: Message>(&mut self, message: &M, now: Instant) {+ self.collector.add(message, now);+ }++ fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {+ if (completion.accepted as usize) < completion.offsets.len() {+ anyhow::bail!(+ "accepted {}/{} messages — not committing offsets",+ completion.accepted,+ completion.offsets.len()+ );+ }+ let Some(topic_partition) = self.uncommitted.topic_partition(completion.partition) else {+ return Ok(());+ };+ let outcome = self.settlement.settle_run(+ topic_partition,+ completion.generation,+ &completion.offsets,+ )?;+ if outcome == RunOutcome::Stale {+ return Ok(());+ }+ counter!("ingestion_consumer_messages_processed_total")+ .increment(completion.offsets.len() as u64);+ self.commit_due = true;+ Ok(())+ }++ fn ticked(&mut self) {+ self.commit_due = true;+ }++ fn revoked(&mut self, topic_partitions: &[TopicPartition]) {+ let building = self.collector.remove_partitions(topic_partitions) as u64;+ self.uncommitted.revoked(topic_partitions);+ self.processed.revoked(topic_partitions);+ report_revoked(building);+ }++ fn shutdown(&mut self) {+ self.accepting_new_batches = false;+ self.commit_due = true;+ self.collector.discard();+ }++ fn submitted(&mut self, poll: InFlightPoll) {+ self.uncommitted.charged(&poll);+ self.processed.submitted(&poll);+ }++ fn next_step(&mut self, now: Instant) -> Option<Step> {+ if std::mem::take(&mut self.commit_due) {+ let commit = if self.accepting_new_batches {+ self.settlement.take_due(now.into_std())+ } else {+ self.settlement.take_all()+ };+ if let Some(commit) = commit {+ self.uncommitted.committed(&commit);+ self.processed.committed(&commit);+ return Some(Step::Commit(commit));+ }+ }+ if let Some(collected) = self.collector.take_ready(now) {+ return Some(Step::Submit(collected));+ }+ (!self.accepting_new_batches && self.uncommitted.is_empty()).then_some(Step::Stop)+ }+}++struct ProcessedTimestamps {+ group_id: String,+ partitions: HashMap<TopicPartition, VecDeque<SubmittedSlice>>,+}++struct SubmittedSlice {+ last_offset: i64,+ latest_kafka_ts: i64,+}++impl ProcessedTimestamps {+ fn new(group_id: String) -> Self {+ Self {+ group_id,+ partitions: HashMap::new(),+ }+ }++ fn submitted(&mut self, poll: &InFlightPoll) {+ for (topic_partition, deliveries) in &poll.partitions {+ self.partitions+ .entry(topic_partition.clone())+ .or_default()+ .push_back(SubmittedSlice {+ last_offset: deliveries.span.last,+ latest_kafka_ts: deliveries.latest_kafka_ts,+ });+ }+ }++ fn committed(&mut self, commit: &Commit) {+ for (topic_partition, next_to_read) in &commit.offsets {+ let Some(polls) = self.partitions.get_mut(topic_partition) else {+ continue;+ };+ let mut latest = None;+ while polls+ .front()+ .is_some_and(|slice| slice.last_offset < next_to_read.0)+ {+ let slice = polls.pop_front().expect("checked above");+ latest = latest.max(Some(slice.latest_kafka_ts));+ }+ if let Some(latest) = latest {+ // Same name as the Node.js consumer's gauge, so one dashboard reads both.+ gauge!(+ "latest_processed_timestamp_ms",+ "topic" => topic_partition.topic.clone(),+ "partition" => topic_partition.partition.to_string(),+ "groupId" => self.group_id.clone()+ )+ .set(latest as f64);+ }+ }+ }++ fn revoked(&mut self, revoked: &[TopicPartition]) {+ for topic_partition in revoked {+ self.partitions.remove(topic_partition);+ }+ }+}++/// Keyed by partition number, because a completion carries no topic.+#[derive(Default)]+struct UncommittedCharges {+ partitions: HashMap<Partition, (TopicPartition, Charge)>,+ total: Charge,+}++impl UncommittedCharges {+ fn total(&self) -> Charge {+ self.total+ }++ fn is_empty(&self) -> bool {+ self.total == Charge::ZERO+ }++ /// `None` once the partition is revoked, until its next charge.+ fn topic_partition(&self, partition: Partition) -> Option<&TopicPartition> {+ self.partitions+ .get(&partition)+ .map(|(topic_partition, _)| topic_partition)+ }++ fn charged(&mut self, poll: &InFlightPoll) {+ for (topic_partition, &charge) in &poll.uncommitted {+ let previous = self.partitions.insert(+ Partition(topic_partition.partition),+ (topic_partition.clone(), charge),+ );+ self.replace(previous.map(|(_, charge)| charge), charge);+ }+ }++ fn committed(&mut self, commit: &Commit) {+ for (topic_partition, &charge) in &commit.uncommitted {+ if let Some((_, partition)) = self+ .partitions+ .get_mut(&Partition(topic_partition.partition))+ {+ let previous = std::mem::replace(partition, charge);+ self.replace(Some(previous), charge);+ }+ }+ }++ fn revoked(&mut self, revoked: &[TopicPartition]) {+ for topic_partition in revoked {+ let previous = self+ .partitions+ .remove(&Partition(topic_partition.partition));+ self.replace(previous.map(|(_, charge)| charge), Charge::ZERO);+ }+ }++ fn replace(&mut self, previous: Option<Charge>, charge: Charge) {+ self.total = self.total - previous.unwrap_or(Charge::ZERO) + charge;+ }+}++#[cfg(test)]+mod tests {+ use std::collections::HashMap;+ use std::num::NonZeroU64;++ use common_kafka_consumer::Offset;+ use metrics_util::debugging::{DebugValue, DebuggingRecorder};++ use super::super::harness::{completion, p, Harness};+ use super::*;++ fn continuous(in_flight_events: u64) -> Harness<ContinuousPolling> {+ Harness::new(|parts| {+ ContinuousPolling::new(+ parts,+ ChargeLimit::new(NonZeroU64::new(in_flight_events), None),+ )+ })+ }++ #[test]+ fn the_continuous_strategy_stops_reading_one_poll_past_its_limit_until_a_commit() {+ let mut harness = continuous(1);++ harness.receive(0, &[0, 1]);+ assert!(+ harness.poller.reading(),+ "the poll in progress is not counted"+ );+ harness.after_the_batch_timeout();+ assert_eq!(harness.submit(), HashMap::from([(p(0), (0, 1))]));+ assert!(!harness.poller.reading());++ harness.complete(0, &[0, 1]);+ assert!(+ !harness.poller.reading(),+ "a completion frees no room before its commit"+ );+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));+ assert!(harness.poller.reading());+ }++ #[test]+ fn a_revoke_removes_the_partitions_messages_received_before_it() {+ let mut harness = continuous(1_000);+ harness.receive(0, &[10, 11]);+ harness.receive(1, &[20]);++ harness.poller.revoked(&[p(0)]);+ harness.receive(0, &[10]);+ harness.after_the_batch_timeout();++ assert_eq!(+ harness.submit(),+ HashMap::from([(p(0), (10, 10)), (p(1), (20, 20))]),+ "the redelivery after the revoke stays"+ );+ }++ #[test]+ fn a_completion_from_before_a_revoke_settles_nothing_in_the_next_assignment() {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);+ let processed = || {+ snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .find_map(|(key, _, _, value)| match value {+ DebugValue::Counter(count)+ if key.key().name() == "ingestion_consumer_messages_processed_total" =>+ {+ Some(count)+ }+ _ => None,+ })+ .unwrap_or_default()+ };+ let mut harness = continuous(1_000);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();+ let before = harness.ledger.generation(&p(0));++ harness.revoke(0);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();++ harness.complete_under(before, 0, &[0, 1]);+ harness.poller.ticked();+ assert!(+ harness.step().is_none(),+ "the old assignment's completion is stale"+ );+ assert_eq!(processed(), 0, "a stale completion processed nothing");+ harness.complete(0, &[0, 1]);+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));+ assert_eq!(processed(), 2);+ }++ #[test]+ fn a_revoke_frees_the_partitions_charge() {+ let mut harness = continuous(3);+ harness.receive(0, &[0, 1, 2]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(!harness.poller.reading());++ harness.revoke(0);+ assert!(+ harness.poller.reading(),+ "a revoked partition holds no charge"+ );++ harness.receive(0, &[0, 1, 2]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(+ !harness.poller.reading(),+ "the new assignment's charge counts in full"+ );+ }++ #[test]+ fn the_limit_sums_each_partitions_uncommitted_charge() {+ let mut harness = continuous(5);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();+ harness.receive(0, &[2, 3]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(+ harness.poller.reading(),+ "a second submit replaces the partition's charge"+ );++ harness.receive(1, &[0]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(+ !harness.poller.reading(),+ "the partitions' charges add up to the limit"+ );++ harness.complete(0, &[0, 1]);+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));+ assert!(+ harness.poller.reading(),+ "a commit frees only the offsets it passed"+ );+ }++ #[test]+ fn a_completion_a_worker_did_not_accept_in_full_fails() {+ let mut harness = continuous(1_000);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();+ let mut short = completion(harness.ledger.generation(&p(0)), 0, &[0, 1]);+ short.accepted = 1;++ let error = harness+ .poller+ .completed(&short)+ .expect_err("a dropped message must not commit");+ assert!(error.to_string().contains("accepted 1/2"), "{error}");+ }++ #[test]+ fn a_commit_reports_the_newest_kafka_timestamp_it_passed() {+ let recorder = DebuggingRecorder::new();+ let snapshotter = recorder.snapshotter();+ let _guard = metrics::set_default_local_recorder(&recorder);+ let latest_processed = || {+ snapshotter+ .snapshot()+ .into_vec()+ .into_iter()+ .find_map(|(key, _, _, value)| match value {+ DebugValue::Gauge(ts)+ if key.key().name() == "latest_processed_timestamp_ms" =>+ {+ Some(ts.into_inner())+ }+ _ => None,+ })+ };+ let mut harness = continuous(1_000);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert_eq!(latest_processed(), None);++ harness.complete(0, &[0, 1]);+ harness.commit();++ assert_eq!(latest_processed(), Some(1_000.0));+ }++ #[test]+ fn shutdown_drops_the_poll_in_progress_and_stops_once_in_flight_work_drains() {+ let mut harness = continuous(1_000);+ harness.receive(0, &[0]);+ harness.after_the_batch_timeout();+ harness.submit();+ harness.receive(0, &[1]);++ harness.poller.shutdown();+ assert!(!harness.poller.reading());+ assert_eq!(harness.poller.deadline(), None);+ assert!(harness.step().is_none(), "offset 0 is still in flight");++ harness.complete(0, &[0]);+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(1))]));+ assert!(matches!(harness.step(), Some(Step::Stop)));+ }+}diff --git a/rust/ingestion-consumer/src/polling/harness.rs b/rust/ingestion-consumer/src/polling/harness.rsnew file mode 100644index 000000000000..51cd6401c958--- /dev/null+++ b/rust/ingestion-consumer/src/polling/harness.rs@@ -0,0 +1,141 @@+use std::collections::HashMap;+use std::sync::Arc;+use std::time::Duration;++use common_kafka_consumer::{+ GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,+};+use tokio::time::Instant;++use super::collector::tests::message;+use super::{CollectedBatch, InFlightPoll, Parts, PollLimits, PollingStrategy, Step};+use crate::partition_budget::BudgetLimits;+use crate::settlement::{GroupSettlement, Settlement};++pub(super) const BATCH_TIMEOUT: Duration = Duration::from_millis(500);++pub(super) struct Harness<P> {+ pub(super) poller: P,+ pub(super) settlement: Arc<dyn Settlement>,+ pub(super) ledger: Arc<TopicOffsetLedger>,+ pub(super) now: Instant,+}++impl<P: PollingStrategy> Harness<P> {+ pub(super) fn new(make: impl FnOnce(Parts) -> P) -> Self {+ let ledger = Arc::new(TopicOffsetLedger::new());+ let settlement: Arc<dyn Settlement> = Arc::new(GroupSettlement::new(+ Arc::clone(&ledger),+ Duration::ZERO,+ BudgetLimits::unset(),+ ));+ let poller = make(Parts {+ limits: PollLimits {+ batch_size: 100,+ batch_size_bytes: 0,+ batch_timeout: BATCH_TIMEOUT,+ },+ settlement: Arc::clone(&settlement),+ group_id: "test".to_string(),+ debug_recorder: None,+ });+ Self {+ poller,+ settlement,+ ledger,+ now: Instant::now(),+ }+ }++ pub(super) fn receive(&mut self, partition: i32, offsets: &[i64]) {+ for &offset in offsets {+ self.poller+ .received(&message(partition, offset, 10), self.now);+ }+ }++ pub(super) fn after_the_batch_timeout(&mut self) {+ self.now += BATCH_TIMEOUT;+ }++ pub(super) fn step(&mut self) -> Option<Step> {+ self.poller.next_step(self.now)+ }++ pub(super) fn submit(&mut self) -> HashMap<TopicPartition, (i64, i64)> {+ let Some(Step::Submit(collected)) = self.step() else {+ panic!("expected a poll to submit");+ };+ let CollectedBatch {+ accumulator,+ mut partitions,+ ..+ } = collected;+ let mut uncommitted = HashMap::new();+ for (topic_partition, deliveries) in &mut partitions {+ deliveries.generation = self.ledger.generation(topic_partition);+ let charged = self+ .settlement+ .charge(topic_partition, deliveries.generation, &deliveries.charges)+ .expect("the ledger accepts the charge");+ uncommitted.insert(topic_partition.clone(), charged.uncommitted);+ }+ let spans = partitions+ .iter()+ .map(|(topic_partition, deliveries)| {+ (+ topic_partition.clone(),+ (deliveries.span.first, deliveries.span.last),+ )+ })+ .collect();+ self.poller.submitted(InFlightPoll {+ poll_id: "poll".to_string(),+ assignment_epoch: 1,+ partitions,+ uncommitted,+ message_count: accumulator.message_count() as u32,+ covered: 0,+ accepted: 0,+ dispatched_at: std::time::Instant::now(),+ });+ spans+ }++ pub(super) fn complete(&mut self, partition: i32, offsets: &[i64]) {+ let generation = self.ledger.generation(&p(partition));+ self.complete_under(generation, partition, offsets);+ }++ pub(super) fn complete_under(&mut self, generation: u64, partition: i32, offsets: &[i64]) {+ self.poller+ .completed(&completion(generation, partition, offsets))+ .expect("the completion applies");+ }++ pub(super) fn revoke(&mut self, partition: i32) {+ self.settlement.forget_partitions(&[p(partition)]);+ self.poller.revoked(&[p(partition)]);+ }++ pub(super) fn commit(&mut self) -> HashMap<TopicPartition, Offset> {+ match self.step() {+ Some(Step::Commit(commit)) => commit.offsets,+ _ => panic!("expected a commit"),+ }+ }+}++pub(super) fn completion(generation: u64, partition: i32, offsets: &[i64]) -> GroupCompletion {+ GroupCompletion {+ partition: Partition(partition),+ assignment_epoch: 1,+ generation,+ offsets: offsets.iter().map(|&offset| Offset(offset)).collect(),+ accepted: offsets.len() as u32,+ }+}++pub(super) fn p(partition: i32) -> TopicPartition {+ TopicPartition::new("events", partition)+}diff --git a/rust/ingestion-consumer/src/polling/mod.rs b/rust/ingestion-consumer/src/polling/mod.rsnew file mode 100644index 000000000000..c0f5efbf1c73--- /dev/null+++ b/rust/ingestion-consumer/src/polling/mod.rs@@ -0,0 +1,197 @@+//! How the consumer loop turns what it receives into what it does next. The+//! consumer owns the Kafka client and the batcher outputs, hands each event to+//! the polling strategy, and carries out the [`Step`]s the strategy returns.++mod capped;+mod collector;+mod continuous;+#[cfg(test)]+mod harness;+mod poll;++use std::sync::Arc;++use common_kafka_consumer::{GroupCompletion, TopicPartition};+use metrics::counter;+use rdkafka::message::Message;+use tokio::time::Instant;+use tracing::{info, warn};++use crate::charge_limit::ChargeLimit;+use crate::config::{CompletionGranularity, Polling};+use crate::debug_recorder::DebugRecorder;+use crate::partition_budget::BudgetLimits;+use crate::settlement::{Commit, Settlement};++pub(crate) use capped::CappedPolling;+use collector::Collector;+pub(crate) use collector::PollLimits;+pub(crate) use continuous::ContinuousPolling;+pub(crate) use poll::{+ debug_partition_offsets, debug_partition_spans, BatchStats, CollectedBatch, InFlightPoll,+ PartitionDeliveries,+};++pub(crate) enum Step {+ /// Charge the poll, submit it to the batcher, and hand it back through+ /// [`PollingStrategy::submitted`].+ Submit(CollectedBatch),+ /// Submit the offsets and resume the partitions.+ Commit(Commit),+ /// Shutdown drained the in-flight work.+ Stop,+ Fail(anyhow::Error),+}++pub(crate) struct Parts {+ pub(crate) limits: PollLimits,+ pub(crate) settlement: Arc<dyn Settlement>,+ pub(crate) group_id: String,+ pub(crate) debug_recorder: Option<Arc<DebugRecorder>>,+}++pub(crate) trait PollingStrategy {+ fn reading(&mut self) -> bool;++ fn deadline(&self) -> Option<Instant>;++ fn received<M: Message>(&mut self, message: &M, now: Instant);++ fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()>;++ fn ticked(&mut self);++ /// Every message received before the revoke belongs to the old+ /// assignment: the batcher purges it, or Kafka delivers it again from the+ /// last commit.+ fn revoked(&mut self, topic_partitions: &[TopicPartition]);++ /// The poll in progress is not charged yet, so it is dropped and replays+ /// from the last commit.+ fn shutdown(&mut self);++ fn submitted(&mut self, poll: InFlightPoll);++ fn next_step(&mut self, now: Instant) -> Option<Step>;+}++fn report_revoked(stripped: u64) {+ if stripped > 0 {+ counter!("ingestion_consumer_polls_dropped_on_revoke_total").increment(stripped);+ info!(stripped, "Removed revoked partitions' messages");+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) enum Selected {+ Capped { max_in_flight_polls: usize },+ Continuous { in_flight: ChargeLimit },+}++/// Continuous polling needs `group` granularity, because at `poll` only a+/// retired poll settles, and continuous polling retires no poll. It needs a+/// partition budget and an in-flight limit, because it has no in-flight poll+/// cap. Each needs both axes, because one axis alone leaves the other+/// unbounded: a message with no key, value or headers charges zero bytes.+/// Without all three the capped strategy runs, so a lane rolled back to `poll` needs no+/// second setting changed.+pub(crate) fn select(+ polling: Polling,+ granularity: CompletionGranularity,+ budget: &BudgetLimits,+ in_flight: ChargeLimit,+ max_in_flight_polls: usize,+) -> Selected {+ let capped = Selected::Capped {+ max_in_flight_polls,+ };+ if polling == Polling::Capped {+ return capped;+ }+ let missing: Vec<&str> = [+ (+ granularity != CompletionGranularity::Group,+ "CONSUMER_COMPLETION_GRANULARITY=group",+ ),+ (+ !budget.sets_both_axes(),+ "a partition budget on events and bytes",+ ),+ (+ !in_flight.sets_both_axes(),+ "an in-flight limit on events and bytes",+ ),+ ]+ .into_iter()+ .filter_map(|(missing, requirement)| missing.then_some(requirement))+ .collect();+ if missing.is_empty() {+ return Selected::Continuous { in_flight };+ }+ warn!(+ missing = missing.join(", "),+ "Running the capped polling strategy: continuous polling needs all its requirements"+ );+ capped+}++#[cfg(test)]+mod tests {+ use std::num::NonZeroU64;++ use super::*;++ #[derive(Clone, Copy)]+ enum Axes {+ Neither,+ Events,+ Bytes,+ Both,+ }++ impl Axes {+ fn limits(self) -> (Option<NonZeroU64>, Option<NonZeroU64>) {+ let (events, bytes) = match self {+ Axes::Neither => (0, 0),+ Axes::Events => (100, 0),+ Axes::Bytes => (0, 100_000),+ Axes::Both => (100, 100_000),+ };+ (NonZeroU64::new(events), NonZeroU64::new(bytes))+ }+ }++ struct SelectCase {+ granularity: CompletionGranularity,+ budget: Axes,+ in_flight: Axes,+ expected_continuous: bool,+ }++ #[rstest::rstest]+ #[case::all_requirements(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Both, in_flight: Axes::Both, expected_continuous: true })]+ #[case::poll_granularity(SelectCase { granularity: CompletionGranularity::Poll, budget: Axes::Both, in_flight: Axes::Both, expected_continuous: false })]+ #[case::no_budget(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Neither, in_flight: Axes::Both, expected_continuous: false })]+ #[case::event_only_budget(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Events, in_flight: Axes::Both, expected_continuous: false })]+ #[case::byte_only_budget(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Bytes, in_flight: Axes::Both, expected_continuous: false })]+ #[case::no_in_flight_limit(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Both, in_flight: Axes::Neither, expected_continuous: false })]+ #[case::event_only_in_flight_limit(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Both, in_flight: Axes::Events, expected_continuous: false })]+ #[case::byte_only_in_flight_limit(SelectCase { granularity: CompletionGranularity::Group, budget: Axes::Both, in_flight: Axes::Bytes, expected_continuous: false })]+ fn continuous_polling_runs_only_with_all_its_requirements(#[case] case: SelectCase) {+ let (events, bytes) = case.budget.limits();+ let budget = BudgetLimits::new(events, bytes, 0.8).unwrap();+ let (events, bytes) = case.in_flight.limits();+ let in_flight = ChargeLimit::new(events, bytes);++ let selected = select(Polling::Continuous, case.granularity, &budget, in_flight, 4);++ let expected = if case.expected_continuous {+ Selected::Continuous { in_flight }+ } else {+ Selected::Capped {+ max_in_flight_polls: 4,+ }+ };+ assert_eq!(selected, expected);+ }+}diff --git a/rust/ingestion-consumer/src/polling/poll.rs b/rust/ingestion-consumer/src/polling/poll.rsnew file mode 100644index 000000000000..e3ddb8c8cf76--- /dev/null+++ b/rust/ingestion-consumer/src/polling/poll.rs@@ -0,0 +1,181 @@+use std::collections::{HashMap, HashSet};+use std::time::Instant;++use common_kafka_consumer::{Charge, Offset, TopicPartition};++use crate::debug_recorder::{PartitionOffset, PartitionOffsets};+use crate::order_sentinel::OffsetSpan;+use crate::settlement::ChargedSlice;+use crate::types::Accumulator;++#[derive(Default)]+pub(crate) struct BatchStats {+ /// One (partition, lag in ms) pair per message with a parseable `now` header.+ pub(crate) message_lags_ms: Vec<(i32, i64)>,+ /// Sum of message payload sizes, without keys or headers.+ pub(crate) total_bytes: usize,+}++pub(crate) struct Delivery {+ pub(crate) offset: i64,+ pub(crate) charge: Charge,+ pub(crate) payload_bytes: usize,+ /// Kafka message timestamp (ms).+ pub(crate) kafka_ts: i64,+ /// Ingestion lag (ms); `None` without a parseable `now` header.+ pub(crate) lag_ms: Option<i64>,+}++pub(crate) struct PartitionDeliveries {+ /// The first and last delivered offsets. A compacted or transactional topic+ /// can skip offsets between them.+ pub(crate) span: OffsetSpan,+ /// The ledger generation, set when the poll is charged.+ pub(crate) generation: u64,+ pub(crate) charges: Vec<(Offset, Charge)>,+ pub(crate) payload_bytes: usize,+ /// Max Kafka message timestamp (ms).+ pub(crate) latest_kafka_ts: i64,+ pub(crate) max_lag_ms: Option<i64>,+ /// Messages the poll delivered, covered, and accepted from this+ /// partition, so a revoked partition can leave the poll exactly.+ pub(crate) delivered: u32,+ pub(crate) covered: u32,+ pub(crate) accepted: u32,+ /// The offsets that completions covered. Keyed by offset, not by position+ /// in `span`, because `span` can have gaps.+ pub(crate) credited: HashSet<i64, ahash::RandomState>,+}++impl PartitionDeliveries {+ pub(crate) fn new(delivery: &Delivery) -> Self {+ Self {+ span: OffsetSpan::new(delivery.offset),+ generation: 0,+ charges: vec![(Offset(delivery.offset), delivery.charge)],+ payload_bytes: delivery.payload_bytes,+ latest_kafka_ts: delivery.kafka_ts,+ max_lag_ms: delivery.lag_ms,+ delivered: 1,+ covered: 0,+ accepted: 0,+ credited: HashSet::default(),+ }+ }++ /// Credit a completion's run and return the offsets to settle: those+ /// charged that no earlier completion credited.+ pub(crate) fn credit_run(&mut self, run: &[Offset]) -> Vec<Offset> {+ let Self {+ credited, charges, ..+ } = self;+ run.iter()+ .copied()+ .filter(|offset| {+ credited.insert(offset.0)+ && charges+ .binary_search_by_key(&offset.0, |(charged, _)| charged.0)+ .is_ok()+ })+ .collect()+ }++ pub(crate) fn uncredited_charge(&self) -> Option<Offset> {+ self.charges+ .iter()+ .map(|(offset, _)| *offset)+ .find(|offset| !self.credited.contains(&offset.0))+ }++ pub(crate) fn record(&mut self, delivery: &Delivery) {+ self.span.extend(delivery.offset);+ self.delivered += 1;+ self.payload_bytes += delivery.payload_bytes;+ self.latest_kafka_ts = self.latest_kafka_ts.max(delivery.kafka_ts);+ if let Some(lag_ms) = delivery.lag_ms {+ self.max_lag_ms = Some(self.max_lag_ms.map_or(lag_ms, |max| max.max(lag_ms)));+ }+ self.charges+ .push((Offset(delivery.offset), delivery.charge));+ }+}++pub(crate) struct CollectedBatch {+ pub(crate) accumulator: Accumulator,+ pub(crate) partitions: HashMap<TopicPartition, PartitionDeliveries>,+ pub(crate) stats: BatchStats,+}++pub(crate) struct InFlightPoll {+ /// For logs and debug events only.+ pub(crate) poll_id: String,+ /// The epoch the batcher stamped on this poll's completions.+ pub(crate) assignment_epoch: u64,+ pub(crate) partitions: HashMap<TopicPartition, PartitionDeliveries>,+ pub(crate) uncommitted: HashMap<TopicPartition, Charge>,+ pub(crate) message_count: u32,+ /// Messages covered by completions so far, accepted or not.+ pub(crate) covered: u32,+ /// Worker-accepted messages so far.+ pub(crate) accepted: u32,+ pub(crate) dispatched_at: Instant,+}++impl InFlightPoll {+ pub(crate) fn is_complete(&self) -> bool {+ self.covered >= self.message_count+ }++ /// A charged offset that no completion covered. A complete poll can still+ /// have one, because `covered` also counts an offset that is credited twice.+ pub(crate) fn uncredited_charge(&self) -> Option<(&TopicPartition, Offset)> {+ self.partitions+ .iter()+ .find_map(|(topic_partition, deliveries)| {+ deliveries+ .uncredited_charge()+ .map(|offset| (topic_partition, offset))+ })+ }++ pub(crate) fn charged_slices(&self) -> Vec<ChargedSlice<'_>> {+ self.partitions+ .iter()+ .map(|(topic_partition, deliveries)| ChargedSlice {+ topic_partition,+ generation: deliveries.generation,+ span: deliveries.span,+ charges: &deliveries.charges,+ })+ .collect()+ }+}++/// Per-partition max offset + observed lag for the debug UI's batch events.+pub(crate) fn debug_partition_offsets(+ partitions: &HashMap<TopicPartition, PartitionDeliveries>,+) -> Vec<PartitionOffset> {+ partitions+ .iter()+ .map(|(topic_partition, partition)| PartitionOffset {+ topic: topic_partition.topic.clone(),+ partition: topic_partition.partition,+ offset: partition.span.last,+ lag_ms: partition.max_lag_ms.unwrap_or(0),+ })+ .collect()+}++/// Per-partition offset ranges, so the debug UI can match requests to polls.+pub(crate) fn debug_partition_spans(+ partitions: &HashMap<TopicPartition, PartitionDeliveries>,+) -> Vec<PartitionOffsets> {+ partitions+ .iter()+ .map(|(topic_partition, partition)| PartitionOffsets {+ topic: topic_partition.topic.clone(),+ partition: topic_partition.partition,+ ranges: vec![(partition.span.first, partition.span.last)],+ })+ .collect()+}diff --git a/rust/ingestion-consumer/src/settlement/group.rs b/rust/ingestion-consumer/src/settlement/group.rsindex e202a0630751..8543ed7329aa 100644--- a/rust/ingestion-consumer/src/settlement/group.rs+++ b/rust/ingestion-consumer/src/settlement/group.rs@@ -4,7 +4,7 @@ use std::time::{Duration, Instant};use common_kafka_consumer::{Charge, Offset, Rejection, TopicOffsetLedger, TopicPartition};use super::{- offsets, Accounting, Admission, ChargedSlice, Commit, PartitionBudget, Pause, Settlement,+ Accounting, Admission, Charged, ChargedSlice, Commit, PartitionBudget, RunOutcome, Settlement,Uncommitted,};use crate::commit_pacer::CommitPacer;@@ -36,11 +36,15 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettletopic_partition: &TopicPartition,generation: u64,charges: &[(Offset, Charge)],- ) -> Result<Option<Pause>, Rejection> {+ ) -> Result<Charged, Rejection> {let uncommitted = self.accounting- .charge(topic_partition, generation, charges)?;- Ok(self.admission.charged(topic_partition, uncommitted.charge))+ .charge(topic_partition, generation, charges)?+ .charge;+ Ok(Charged {+ uncommitted,+ pause: self.admission.charged(topic_partition, uncommitted),+ })}fn settle_run(@@ -48,9 +52,9 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettletopic_partition: &TopicPartition,generation: u64,offsets: &[Offset],- ) -> anyhow::Result<()> {+ ) -> anyhow::Result<RunOutcome> {let (Some(first), Some(last)) = (offsets.first(), offsets.last()) else {- return Ok(());+ return Ok(RunOutcome::Settled);};if let Err(rejection) =self.accounting@@ -65,8 +69,9 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettlelast: last.0,}),)?;+ return Ok(RunOutcome::Stale);}- Ok(())+ Ok(RunOutcome::Settled)}fn settle_poll(&self, _slices: &[ChargedSlice<'_>]) -> anyhow::Result<()> {@@ -75,18 +80,14 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettlefn take_due(&self, now: Instant) -> Option<Commit> {let handed_out = self.accounting.take_due(now)?;- Some(Commit {- resume: self.admission.handed_out(&handed_out),- offsets: offsets(handed_out),- })+ let resume = self.admission.handed_out(&handed_out);+ Some(Commit::new(handed_out, resume))}fn take_all(&self) -> Option<Commit> {let handed_out = self.accounting.take_all()?;- Some(Commit {- resume: self.admission.handed_out(&handed_out),- offsets: offsets(handed_out),- })+ let resume = self.admission.handed_out(&handed_out);+ Some(Commit::new(handed_out, resume))}fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {@@ -102,7 +103,7 @@ mod tests {use common_kafka_consumer::{Held, LedgerError};use super::super::fakes::{FakeAccounting, FakeAdmission, Settled};- use super::super::HandedOut;+ use super::super::{HandedOut, Pause};use super::*;fn p(partition: i32) -> TopicPartition {@@ -147,7 +148,12 @@ mod tests {accounting.charge_returns(Ok(uncommitted_events(10)));admission.charged_returns(Some(Pause));- assert_eq!(settlement.charge(&p(0), 0, &[]), Ok(Some(Pause)));+ assert_eq!(+ settlement+ .charge(&p(0), 0, &[])+ .map(|charged| charged.pause),+ Ok(Some(Pause))+ );assert_eq!(admission.charges_seen(), vec![(p(0), events(10))]);}@@ -160,7 +166,12 @@ mod tests {};accounting.charge_returns(Err(stale));- assert_eq!(settlement.charge(&p(0), 0, &[]), Err(stale));+ assert_eq!(+ settlement+ .charge(&p(0), 0, &[])+ .map(|charged| charged.pause),+ Err(stale)+ );assert!(admission.charges_seen().is_empty());}@@ -285,7 +296,7 @@ mod integration_tests {use std::collections::HashMap;use std::num::NonZeroU64;- use super::super::charge_offsets;+ use super::super::{charge_offsets, Pause};use super::*;fn settlement() -> (Arc<TopicOffsetLedger>, GroupSettlement) {@@ -449,6 +460,7 @@ mod integration_tests {&[(Offset(offset), Charge { events, bytes: 0 })],).expect("the ledger accepts the charge")+ .pause}#[test]diff --git a/rust/ingestion-consumer/src/settlement/mod.rs b/rust/ingestion-consumer/src/settlement/mod.rsindex 2bbf4d15608f..65b3328fc3bd 100644--- a/rust/ingestion-consumer/src/settlement/mod.rs+++ b/rust/ingestion-consumer/src/settlement/mod.rs@@ -37,7 +37,7 @@ pub(crate) trait Settlement: Send + Sync {topic_partition: &TopicPartition,generation: u64,charges: &[(Offset, Charge)],- ) -> Result<Option<Pause>, Rejection>;+ ) -> Result<Charged, Rejection>;/// Accepted offsets from one poll's slice of a partition, charged under `generation`.fn settle_run(@@ -45,7 +45,7 @@ pub(crate) trait Settlement: Send + Sync {topic_partition: &TopicPartition,generation: u64,offsets: &[Offset],- ) -> anyhow::Result<()>;+ ) -> anyhow::Result<RunOutcome>;/// The oldest poll retired with every message accepted.fn settle_poll(&self, slices: &[ChargedSlice<'_>]) -> anyhow::Result<()>;@@ -88,19 +88,42 @@ pub(crate) trait Admission: Send + Sync {fn forget_partitions(&self, topic_partitions: &[TopicPartition]);}+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(crate) enum RunOutcome {+ Settled,+ /// Charged under a generation that a rebalance has since replaced.+ Stale,+}+#[derive(Debug, PartialEq, Eq)]pub(crate) struct Pause;+#[derive(Debug, PartialEq, Eq)]+pub(crate) struct Charged {+ pub(crate) uncommitted: Charge,+ pub(crate) pause: Option<Pause>,+}+pub(crate) struct Commit {pub(crate) offsets: HashMap<TopicPartition, Offset>,+ pub(crate) uncommitted: HashMap<TopicPartition, Charge>,pub(crate) resume: Vec<TopicPartition>,}-fn offsets(handed_out: HashMap<TopicPartition, HandedOut>) -> HashMap<TopicPartition, Offset> {- handed_out- .into_iter()- .map(|(topic_partition, handed_out)| (topic_partition, handed_out.offset))- .collect()+impl Commit {+ fn new(handed_out: HashMap<TopicPartition, HandedOut>, resume: Vec<TopicPartition>) -> Self {+ let mut offsets = HashMap::with_capacity(handed_out.len());+ let mut uncommitted = HashMap::with_capacity(handed_out.len());+ for (topic_partition, handed_out) in handed_out {+ uncommitted.insert(topic_partition.clone(), handed_out.remaining.charge);+ offsets.insert(topic_partition, handed_out.offset);+ }+ Self {+ offsets,+ uncommitted,+ resume,+ }+ }}pub(crate) fn for_granularity(diff --git a/rust/ingestion-consumer/src/settlement/poll.rs b/rust/ingestion-consumer/src/settlement/poll.rsindex 7346a7b53568..b73483988361 100644--- a/rust/ingestion-consumer/src/settlement/poll.rs+++ b/rust/ingestion-consumer/src/settlement/poll.rs@@ -5,7 +5,7 @@ use common_kafka_consumer::{Charge, Offset, Rejection, TopicOffsetLedger, TopicPuse metrics::counter;use tracing::warn;-use super::{offsets, Accounting, ChargedSlice, Commit, Pause, Settlement, Uncommitted};+use super::{Accounting, Charged, ChargedSlice, Commit, RunOutcome, Settlement, Uncommitted};use crate::commit_pacer::CommitPacer;use crate::ledger_rejection::{report_rejection, RejectedSlice};@@ -27,10 +27,13 @@ impl<A: Accounting + 'static> Settlement for PollSettlement<A> {topic_partition: &TopicPartition,generation: u64,charges: &[(Offset, Charge)],- ) -> Result<Option<Pause>, Rejection> {+ ) -> Result<Charged, Rejection> {self.uncommitted.charge(topic_partition, generation, charges)- .map(|_| None)+ .map(|uncommitted| Charged {+ uncommitted: uncommitted.charge,+ pause: None,+ })}fn settle_run(@@ -38,8 +41,8 @@ impl<A: Accounting + 'static> Settlement for PollSettlement<A> {_topic_partition: &TopicPartition,_generation: u64,_offsets: &[Offset],- ) -> anyhow::Result<()> {- Ok(())+ ) -> anyhow::Result<RunOutcome> {+ Ok(RunOutcome::Settled)}fn settle_poll(&self, slices: &[ChargedSlice<'_>]) -> anyhow::Result<()> {@@ -61,18 +64,12 @@ impl<A: Accounting + 'static> Settlement for PollSettlement<A> {fn take_due(&self, now: Instant) -> Option<Commit> {let handed_out = self.uncommitted.take_due(now)?;- Some(Commit {- offsets: offsets(handed_out),- resume: Vec::new(),- })+ Some(Commit::new(handed_out, Vec::new()))}fn take_all(&self) -> Option<Commit> {let handed_out = self.uncommitted.take_all()?;- Some(Commit {- offsets: offsets(handed_out),- resume: Vec::new(),- })+ Some(Commit::new(handed_out, Vec::new()))}fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {@@ -307,7 +304,12 @@ mod tests {},}));- assert_eq!(settlement.charge(&p(0), 2, &[]), Ok(None));+ assert_eq!(+ settlement+ .charge(&p(0), 2, &[])+ .map(|charged| charged.pause),+ Ok(None)+ );}#[test]diff --git a/rust/ingestion-consumer/src/transport.rs b/rust/ingestion-consumer/src/transport.rsindex f7fee8c4f385..5372dc716d25 100644--- a/rust/ingestion-consumer/src/transport.rs+++ b/rust/ingestion-consumer/src/transport.rs@@ -10,8 +10,8 @@ use crate::types::SerializedKafkaMessage;pub const DEFAULT_MAX_BODY_BYTES: usize = 10 * 1024 * 1024;/// Approximate serialized size of one message within a frame. Field names and-/// punctuation are a small constant; escaping of the embedded JSON text is-/// absorbed by the headroom between the split cap and the worker's hard limit.+/// punctuation are a small constant; JSON string escaping is ignored (see+/// [`DEFAULT_MAX_BODY_BYTES`]).fn approx_message_size(msg: &SerializedKafkaMessage) -> usize {const PER_MESSAGE_OVERHEAD: usize = 96;msg.topic.len()@@ -62,9 +62,8 @@ pub enum SendOutcome {},}-/// Failure from a worker stream send, carrying back the sub-batch's messages-/// so the caller can requeue and replay them (the worker may have died-/// mid-send and the messages were never accepted).+/// Failure from a worker stream send. It hands back every message of the+/// sub-batch, in send order, so the caller can requeue and replay them.#[derive(Debug)]pub struct SendError {pub error: TransportError,@@ -124,9 +123,6 @@ pub enum TransportError {}impl TransportError {- /// Worker stream failures resolve through the deferral path and are never- /// retried in place. A busy worker stream is transient backpressure, so the- /// fenced work re-routes as retriable rather than as a worker fault.pub fn is_retriable(&self) -> bool {matches!(self, TransportError::WorkerStreamBusy(_))}@@ -161,6 +157,7 @@ mod tests {key: None,value: Some("x".repeat(value_len)),headers: std::collections::HashMap::new(),+ generation: 0,}}diff --git a/rust/ingestion-consumer/src/types.rs b/rust/ingestion-consumer/src/types.rsindex 948d747581a2..83d48ba0153e 100644--- a/rust/ingestion-consumer/src/types.rs+++ b/rust/ingestion-consumer/src/types.rs@@ -15,6 +15,10 @@ pub struct SerializedKafkaMessage {pub key: Option<String>,pub value: Option<String>,pub headers: HashMap<String, String>,+ /// The ledger generation the message was charged under. Workers never+ /// see it.+ #[serde(skip)]+ pub generation: u64,}impl SerializedKafkaMessage {diff --git a/rust/ingestion-consumer/tests/batcher_integration_test.rs b/rust/ingestion-consumer/tests/batcher_integration_test.rsindex 74fe73a6cd7d..5d99506a40d1 100644--- a/rust/ingestion-consumer/tests/batcher_integration_test.rs+++ b/rust/ingestion-consumer/tests/batcher_integration_test.rs@@ -36,6 +36,7 @@ fn make_msg(key: &str) -> SerializedKafkaMessage {key: Some(key.to_string()),value: None,headers: HashMap::new(),+ generation: 0,}}@@ -56,9 +57,8 @@ async fn purging_a_just_submitted_batch_is_not_fatal() {let mut accumulator = Accumulator::default();accumulator.push(Partition(0), make_msg("a").into());- batcher.submit(accumulator);- // No await between submit and purge: the batcher task receives both- // before it runs, and discards the submission when it applies the purge.+ batcher.submit(accumulator, &HashMap::new());+ // No await, so the task gets the submit and the purge before it runs.batcher.revoker().purge_revoked(&[("test".to_string(), 0)]);match tokio::time::timeout(Duration::from_millis(100), outputs.errors.recv()).await {diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rsindex 75a3c01641d3..caffb54f2680 100644--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs@@ -1,7 +1,4 @@-//! End-to-end integration tests for the ingestion consumer pipeline.-//!-//! Requires Kafka on localhost:9092 (available via docker-compose).-//! Each test creates a uniquely-named topic to avoid cross-test interference.+//! Requires Kafka on localhost:9092 (docker-compose).mod common;@@ -34,8 +31,10 @@ use ingestion_consumer::batcher::packer::PackTargets;use ingestion_consumer::batcher::state_machine::RequestBudget;use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;use ingestion_consumer::batcher::BatcherObserver;-use ingestion_consumer::config::CompletionGranularity;+use ingestion_consumer::charge_limit::ChargeLimit;+use ingestion_consumer::config::{CompletionGranularity, Polling};use ingestion_consumer::consumer::{IngestionConsumer, IngestionConsumerOptions};+use ingestion_consumer::debug_recorder::{DebugEventKind, DebugRecorder};use ingestion_consumer::discovery::reconcile_membership;use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport};use ingestion_consumer::partition_budget::BudgetLimits;@@ -52,11 +51,10 @@ use ingestion_worker_proto::ingestion::worker::v1::{MessageRef, StreamReady, SubBatchAck, SubBatchStatus,};+const UNBOUNDED_BYTES: u64 = 1 << 40;const KAFKA_BROKERS: &str = "localhost:9092";const BUDGET_ATTEMPTS: NonZeroU32 = NonZeroU32::new(3).expect("nonzero attempts");-// ── Kafka helpers ──────────────────────────────────────────────────────────-async fn create_topic(name: &str, partitions: i32) {let admin: AdminClient<DefaultClientContext> = ClientConfig::new().set("bootstrap.servers", KAFKA_BROKERS)@@ -114,10 +112,6 @@ async fn produce(.expect("produce failed");}-/// Produce `(distinct_id, seq)` records to one partition in one transaction.-/// The consumer reads committed records only, so it sees all of them at once-/// and fetches them into one batch. Awaited one-by-one sends can straddle the-/// consumer's batch deadline and split the records across batches.async fn produce_in_one_transaction(topic: &str,partition: i32,@@ -135,8 +129,6 @@ async fn produce_in_one_transaction(produce_keyed_in_one_transaction(topic, token, &records).await;}-/// Visitors A and B take turns behind one Kafka key, as cookieless visitors-/// behind one `token:ip` key do.fn alternating_visitors(partition: i32,key: &str,@@ -150,8 +142,6 @@ fn alternating_visitors(]}-/// Like `produce_in_one_transaction`, with each record given as-/// `(partition, key, distinct_id, seq)`.async fn produce_keyed_in_one_transaction(topic: &str,token: &str,@@ -209,8 +199,6 @@ async fn produce_keyed_in_one_transaction(.expect("transactional produce panicked");}-/// Produce a record with full control over payload bytes and optional headers —-/// for malformed-input scenarios (missing headers, non-UTF-8 bytes).async fn produce_raw(producer: &FutureProducer,topic: &str,@@ -232,28 +220,18 @@ async fn produce_raw(.expect("produce failed");}-// ── Fake worker ────────────────────────────────────────────────────────────-struct FakeWorker {pub url: String,- /// (distinct_id, seq) pairs in arrival order.pub received: Arc<Mutex<Vec<(String, usize)>>>,- /// Message count of each acked sub-batch, in arrival order. With a- /// single worker and partition a sub-batch is the whole Kafka batch, which- /// is what lets the batching-bound tests read batch sizes from here.+ /// Message count of each recorded sub-batch, in arrival order.pub batch_sizes: Arc<Mutex<Vec<usize>>>,- /// Gates /_ready (pool membership). When false the worker leaves the pool.+ /// When false, /_ready fails and every send nacks.pub healthy: Arc<AtomicBool>,- /// Gates ingest only. When false the worker stays ready (in the pool) but- /// fails every send — a "flapping" worker, used to exercise the flush loop's- /// re-defer path and its timeout bound.+ /// When false, the worker stays ready but nacks every send.pub ingest_ok: Arc<AtomicBool>,- /// When set, the next sub-batch is processed and recorded but answered- /// with a nack — a "lost ACK". Models the ambiguous outcome where the- /// worker did the work but the consumer never learns it, so it replays.+ /// When set, the next sub-batch is recorded, then nacked, so its replay duplicates it.pub ack_lost_once: Arc<AtomicBool>,- /// When true, every sub-batch is nacked without- /// ingesting anything — a poison batch this worker will never accept.+ /// When true, every sub-batch is nacked unrecorded.pub reject_4xx: Arc<AtomicBool>,/// When set to `(distinct_id, frames)`, the next `frames` sub-batches with/// a soft budget that carry this distinct_id leave all of that id's@@ -261,35 +239,26 @@ struct FakeWorker {pub leave_unfinished: Arc<Mutex<Option<(String, usize)>>>,/// (distinct_id, seq) pairs that arrived in a sub-batch marked as replay.pub replayed: Arc<Mutex<Vec<(String, usize)>>>,- /// When true, a replay sub-batch is dropped without an ack, as a worker- /// stuck on it would leave it.+ /// When true, replay sub-batches get no answer and stay in flight.pub hold_replays: Arc<AtomicBool>,- /// When set, the next sub-batch is ingested in full but reports one- /// message fewer than sent — a partial-acceptance contract violation.+ /// When set, the next sub-batch is recorded in full but acked as one message fewer.pub underreport_once: Arc<AtomicBool>,- /// Number of sub-batches that have reached the handler. Counted before- /// the gate, so a test can observe a batch is in-flight even while held.arrived: Arc<AtomicUsize>,- /// Held by the handler for the duration of each sub-batch. A test can- /// acquire it via `block()` to freeze a request in flight (simulating a slow- /// worker), then drop the guard to let it complete.+ on_arrival: ArrivalHook,gate: Arc<tokio::sync::Mutex<()>>,_task: tokio::task::JoinHandle<()>,}-/// A receipt-ordered log of `(distinct_id, seq)` shared across all workers in a-/// run. The shared mutex gives every accepted message a slot in a single total-/// order, so a test can reconstruct global delivery order across reroutes and-/// assert per-distinct_id ordering, no loss, and no duplication.+/// Shared by all workers, so its mutex puts every accepted message in one total order.type DeliveryLog = Arc<Mutex<Vec<(String, usize)>>>;+type ArrivalHook = Arc<Mutex<Option<Box<dyn Fn() + Send + Sync>>>>;+impl FakeWorker {async fn start() -> Self {Self::start_inner(None).await}- /// Like `start`, but also appends every accepted message to a shared,- /// cross-worker delivery log (for the churn/ordering suite).async fn start_logged(delivery_log: DeliveryLog) -> Self {Self::start_inner(Some(delivery_log)).await}@@ -306,6 +275,7 @@ impl FakeWorker {let replayed = Arc::new(Mutex::new(Vec::new()));let hold_replays = Arc::new(AtomicBool::new(false));let arrived = Arc::new(AtomicUsize::new(0));+ let on_arrival: ArrivalHook = Arc::new(Mutex::new(None));let gate = Arc::new(tokio::sync::Mutex::new(()));let app = Router::new().route(@@ -325,9 +295,7 @@ impl FakeWorker {}),);- // Bind HTTP on an ephemeral port and gRPC on that port + 1, so the- // gRPC transport's OffsetFromHttp mode can address each worker.- // Rebind both if the adjacent port happens to be taken.+ // gRPC listens on the HTTP port + 1, as `GrpcPort::OffsetFromHttp(1)` expects.let (listener, grpc_listener) = loop {let http = TcpListener::bind("127.0.0.1:0").await.unwrap();let Some(grpc_port) = http.local_addr().unwrap().port().checked_add(1) else {@@ -354,6 +322,7 @@ impl FakeWorker {replayed: Arc::clone(&replayed),hold_replays: Arc::clone(&hold_replays),arrived: Arc::clone(&arrived),+ on_arrival: Arc::clone(&on_arrival),gate: Arc::clone(&gate),delivery_log,};@@ -384,6 +353,7 @@ impl FakeWorker {replayed,hold_replays,arrived,+ on_arrival,gate,_task: task,}@@ -393,7 +363,6 @@ impl FakeWorker {self.received.lock().unwrap().len()}- /// Message count of each acked sub-batch, in arrival order.fn batch_sizes(&self) -> Vec<usize> {self.batch_sizes.lock().unwrap().clone()}@@ -409,6 +378,11 @@ impl FakeWorker {Arc::clone(&self.gate).lock_owned().await}+ /// Run `hook` as each sub-batch arrives, before the gate holds it.+ fn on_arrival(&self, hook: impl Fn() + Send + Sync + 'static) {+ *self.on_arrival.lock().unwrap() = Some(Box::new(hook));+ }+fn seqs_for(&self, distinct_id: &str) -> Vec<usize> {self.received.lock()@@ -420,8 +394,7 @@ impl FakeWorker {}}-/// The FakeWorker's WorkerIngest implementation. Per the protocol, any-/// failure ends the stream (a nack means the worker stream is dead).+// A nack ends the stream, as the protocol requires.struct FakeWorkerGrpc {received: Arc<Mutex<Vec<(String, usize)>>>,batch_sizes: Arc<Mutex<Vec<usize>>>,@@ -434,6 +407,7 @@ struct FakeWorkerGrpc {replayed: Arc<Mutex<Vec<(String, usize)>>>,hold_replays: Arc<AtomicBool>,arrived: Arc<AtomicUsize>,+ on_arrival: ArrivalHook,gate: Arc<tokio::sync::Mutex<()>>,delivery_log: Option<DeliveryLog>,}@@ -461,6 +435,7 @@ impl WorkerIngestService for FakeWorkerGrpc {let replayed = Arc::clone(&self.replayed);let hold_replays = Arc::clone(&self.hold_replays);let arrived = Arc::clone(&self.arrived);+ let on_arrival = Arc::clone(&self.on_arrival);let gate = Arc::clone(&self.gate);let delivery_log = self.delivery_log.clone();@@ -490,7 +465,9 @@ impl WorkerIngestService for FakeWorkerGrpc {continue;};arrived.fetch_add(1, Ordering::SeqCst);- // Block here while a test holds the gate (slow worker).+ if let Some(hook) = on_arrival.lock().unwrap().as_ref() {+ hook();+ }let _hold = gate.lock().await;if !healthy.load(Ordering::Relaxed) || !ingest_ok.load(Ordering::Relaxed) {nack(&tx, sub_batch.seq, "worker unhealthy");@@ -567,8 +544,6 @@ impl WorkerIngestService for FakeWorkerGrpc {}));continue;}- // Work done and recorded, but the ACK is lost: fail the stream- // so the consumer fences and replays.if ack_lost_once.swap(false, Ordering::SeqCst) {nack(&tx, sub_batch.seq, "ack lost");return;@@ -597,36 +572,31 @@ impl WorkerIngestService for FakeWorkerGrpc {}}-// ── Test harness ───────────────────────────────────────────────────────────-const COMMIT_INTERVAL: Duration = Duration::from_millis(500);struct Harness {pub workers: Vec<FakeWorker>,pub registry: Arc<WorkerRegistry>,pub observer: BatcherObserver,- /// Cross-worker, receipt-ordered log of every accepted message.pub delivery_log: DeliveryLog,pub shutdown: CancellationToken,task: Option<tokio::task::JoinHandle<()>>,_probe_token: CancellationToken,topic: String,- /// Consumer group id, kept so `restart_consumer` rejoins the same group.group_id: String,- /// The running consumer's offset ledger, for observing its generations.pub ledger: Arc<TopicOffsetLedger>,+ debug_recorder: Arc<DebugRecorder>,max_in_flight: usize,deferred_flush_timeout: Duration,pack_targets: PackTargets,budget: Option<RequestBudget>,completion_granularity: CompletionGranularity,+ partition_budget: BudgetLimits,+ polling: Polling,+ in_flight_limit: ChargeLimit,}-/// Build a Kafka consumer subscribed to `topic` in `group_id`, configured like-/// the production batch consumer (no auto commit/store, earliest reset). The-/// short session timeout and heartbeat make group handovers observable-/// quickly in tests. `instance_id` opts into static membership-/// (`group.instance.id`).+// Short session and heartbeat so group handovers finish quickly.fn make_kafka_consumer(topic: &str,group_id: &str,@@ -635,7 +605,6 @@ fn make_kafka_consumer(make_kafka_consumer_with_context(topic, group_id, instance_id, RebalanceContext::new())}-/// Like `make_kafka_consumer`, under a context the test keeps a hand on.fn make_kafka_consumer_with_context(topic: &str,group_id: &str,@@ -661,18 +630,12 @@ fn make_kafka_consumer_with_context(kafka_consumer}-/// The consumer's worker transport, addressed at each FakeWorker's gRPC port-/// (`GrpcPort::OffsetFromHttp(1)` — HTTP port + 1), with a short ack timeout-/// so a stalled scenario fences within a test's patience rather than-/// production's 60s.+/// Each FakeWorker serves gRPC on its HTTP port + 1; a 10 s ack timeout (production: 60 s) so a stalled request fences quickly.fn test_transport() -> GrpcTransport {GrpcTransport::new(GrpcPort::OffsetFromHttp(1), Duration::from_secs(10))}-/// Reap drained workers exactly as `main.rs` does in production: complete the-/// drain of an idle drainer, then remove reaped workers from the registry and-/// transport. Runs until `token` is cancelled. (A faster tick than production's-/// 1s keeps test scenarios responsive.)+/// Ticks every 100 ms, against 1 s in production.fn spawn_reaper(registry: Arc<WorkerRegistry>,transport: Arc<GrpcTransport>,@@ -721,12 +684,13 @@ impl Harness {CompletionGranularity::Poll,COMMIT_INTERVAL,BudgetLimits::unset(),+ Polling::Capped,+ ChargeLimit::unset(),).await}- /// One key-table consumer with one worker, sending a soft budget on every- /// request except an escalated key's resend.+ /// One worker; every request carries a 500 ms soft budget.async fn start_budgeted(topic: &str, partitions: i32) -> Self {Self::start_inner(topic,@@ -754,6 +718,8 @@ impl Harness {CompletionGranularity::Poll,COMMIT_INTERVAL,BudgetLimits::unset(),+ Polling::Capped,+ ChargeLimit::unset(),).await}@@ -782,6 +748,8 @@ impl Harness {CompletionGranularity::Poll,COMMIT_INTERVAL,BudgetLimits::unset(),+ Polling::Capped,+ ChargeLimit::unset(),).await}@@ -809,32 +777,61 @@ impl Harness {CompletionGranularity::Poll,COMMIT_INTERVAL,BudgetLimits::unset(),+ Polling::Capped,+ ChargeLimit::unset(),).await}- /// Like `start`, but bounds batch collection by payload bytes as well as- /// count (`CONSUMER_BATCH_SIZE_KB`). Single worker and partition, so each- /// sub-batch is one whole Kafka batch.- async fn start_byte_capped(topic: &str, batch_size_bytes: usize) -> Self {+ async fn start_polling(+ topic: &str,+ partitions: i32,+ worker_count: usize,+ max_in_flight: usize,+ batch_size_bytes: usize,+ polling: Polling,+ ) -> Self {+ let (granularity, partition_budget, in_flight_limit) = match polling {+ Polling::Capped => (+ CompletionGranularity::Poll,+ BudgetLimits::unset(),+ ChargeLimit::unset(),+ ),+ Polling::Continuous => (+ CompletionGranularity::Group,+ BudgetLimits::new(+ NonZeroU64::new(10_000),+ NonZeroU64::new(UNBOUNDED_BYTES),+ 0.8,+ )+ .expect("a valid budget"),+ ChargeLimit::new(NonZeroU64::new(100_000), NonZeroU64::new(UNBOUNDED_BYTES)),+ ),+ };Self::start_inner(topic,- 1,- 1,- 1,+ partitions,+ worker_count,+ max_in_flight,Duration::from_secs(60),fast_registry_config(),batch_size_bytes,ComponentOptions::new(),common::PRODUCTION_PACK_TARGETS,None,- CompletionGranularity::Poll,+ granularity,COMMIT_INTERVAL,- BudgetLimits::unset(),+ partition_budget,+ polling,+ in_flight_limit,).await}+ async fn start_byte_capped(topic: &str, batch_size_bytes: usize, polling: Polling) -> Self {+ Self::start_polling(topic, 1, 1, 1, batch_size_bytes, polling).await+ }+async fn start_group_granularity(pack_targets: PackTargets,topic: &str,@@ -875,6 +872,8 @@ impl Harness {CompletionGranularity::Group,commit_interval,BudgetLimits::unset(),+ Polling::Capped,+ ChargeLimit::unset(),).await}@@ -885,14 +884,60 @@ impl Harness {partitions: i32,worker_count: usize,partition_budget: BudgetLimits,+ polling: Polling,) -> Self {- Self::start_with_partition_budget_committing_every(- pack_targets,+ let (max_in_flight, in_flight_limit) = match polling {+ Polling::Capped => (8, ChargeLimit::unset()),+ // One poll slot: the continuous loop must not wait on any poll.+ Polling::Continuous => (+ 1,+ ChargeLimit::new(NonZeroU64::new(1_000), NonZeroU64::new(UNBOUNDED_BYTES)),+ ),+ };+ Self::start_inner(topic,partitions,worker_count,+ max_in_flight,+ Duration::from_secs(60),+ fast_registry_config(),+ 0,+ ComponentOptions::new(),+ pack_targets,+ None,+ CompletionGranularity::Group,+ COMMIT_INTERVAL,partition_budget,+ polling,+ in_flight_limit,+ )+ .await+ }++ async fn start_continuous(+ pack_targets: PackTargets,+ topic: &str,+ partitions: i32,+ worker_count: usize,+ partition_budget: BudgetLimits,+ in_flight_limit: ChargeLimit,+ ) -> Self {+ Self::start_inner(+ topic,+ partitions,+ worker_count,+ 1,+ Duration::from_secs(60),+ fast_registry_config(),+ 0,+ ComponentOptions::new(),+ pack_targets,+ None,+ CompletionGranularity::Group,COMMIT_INTERVAL,+ partition_budget,+ Polling::Continuous,+ in_flight_limit,).await}@@ -919,6 +964,8 @@ impl Harness {CompletionGranularity::Group,commit_interval,partition_budget,+ Polling::Capped,+ ChargeLimit::unset(),).await}@@ -938,6 +985,8 @@ impl Harness {completion_granularity: CompletionGranularity,commit_interval: Duration,partition_budget: BudgetLimits,+ polling: Polling,+ in_flight_limit: ChargeLimit,) -> Self {create_topic(topic, partitions).await;@@ -966,6 +1015,7 @@ impl Harness {let group_id = format!("e2e-{}", Uuid::new_v4());let ledger = Arc::new(TopicOffsetLedger::new());+ let debug_recorder = DebugRecorder::new(10_000, Duration::from_secs(3600));let kafka_consumer =make_kafka_consumer_with_context(topic, &group_id, None, RebalanceContext::new());@@ -989,11 +1039,13 @@ impl Harness {batch_timeout: Duration::from_millis(100),max_in_flight_batches: max_in_flight,group_id: "e2e-test".to_string(),- debug_recorder: None,+ debug_recorder: Some(Arc::clone(&debug_recorder)),completion_granularity,commit_interval,ledger: Arc::clone(&ledger),partition_budget,+ polling,+ in_flight_limit,},handle,);@@ -1021,26 +1073,38 @@ impl Harness {topic: topic.to_string(),group_id,ledger,+ debug_recorder,max_in_flight,deferred_flush_timeout,pack_targets,budget,completion_granularity,+ partition_budget,+ polling,+ in_flight_limit,}}- /// Simulate a crash: abort the consumer task in place — no shutdown signal,- /// no draining, no offset commit — as an OOM-kill would.+ /// Message count of each poll the consumer dispatched, before the packer merges any.+ fn poll_sizes(&self) -> Vec<usize> {+ self.debug_recorder+ .backlog()+ .into_iter()+ .filter_map(|event| match event.kind {+ DebugEventKind::BatchDispatched { messages, .. } => Some(messages),+ _ => None,+ })+ .collect()+ }++ /// Abort the consumer task: no shutdown, no drain, no final commit.fn crash_consumer(&mut self) {if let Some(task) = self.task.take() {task.abort();}}- /// Start a fresh consumer stack — a new Kafka client in the same consumer- /// group, plus new registry/batcher/transport — against the same topic- /// and workers, modeling a pod restart. Only the FakeWorkers survive,- /// mirroring production where worker pods outlive a consumer pod.+ /// Model a pod restart: a new consumer stack in the same group, against the same workers.async fn restart_consumer(&mut self, registry_config: WorkerRegistryConfig) {let worker_urls: Vec<String> = self.workers.iter().map(|w| w.url.clone()).collect();@@ -1086,7 +1150,9 @@ impl Harness {completion_granularity: self.completion_granularity,commit_interval: COMMIT_INTERVAL,ledger: Arc::clone(&self.ledger),- partition_budget: BudgetLimits::unset(),+ partition_budget: self.partition_budget,+ polling: self.polling,+ in_flight_limit: self.in_flight_limit,},handle,);@@ -1105,8 +1171,6 @@ impl Harness {tokio::time::sleep(Duration::from_millis(300)).await;}- /// Wait for the consumer's process loop to exit on its own (e.g. after it- /// fails a batch), up to `timeout`. Returns true if it exited within the bound.async fn wait_for_consumer_exit(&mut self, timeout: Duration) -> bool {let Some(task) = self.task.take() else {return true;@@ -1114,8 +1178,6 @@ impl Harness {tokio::time::timeout(timeout, task).await.is_ok()}- /// The group's committed offset for one partition of the harness topic;- /// `None` while nothing is committed.fn committed_offset(&self, partition: i32) -> Option<i64> {broker_committed_offset(&self.group_id, &self.topic, partition)}@@ -1142,8 +1204,7 @@ impl Harness {}}- /// A second consumer in the same group, configured like the first, with- /// its own routing stack and the same worker pool, as a second pod would be.+ /// A second consumer in the same group, over the same workers, as a second pod.async fn join_second_consumer(&self) -> SecondConsumer {let worker_urls: Vec<String> = self.workers.iter().map(|w| w.url.clone()).collect();let registry = Arc::new(WorkerRegistry::new(&worker_urls, fast_registry_config()));@@ -1178,7 +1239,9 @@ impl Harness {completion_granularity: self.completion_granularity,commit_interval: COMMIT_INTERVAL,ledger: Arc::new(TopicOffsetLedger::new()),- partition_budget: BudgetLimits::unset(),+ partition_budget: self.partition_budget,+ polling: self.polling,+ in_flight_limit: self.in_flight_limit,},handle,);@@ -1221,7 +1284,6 @@ fn fast_registry_config() -> WorkerRegistryConfig {}}-/// Poll `cond` until it returns true or the timeout elapses (panics on timeout).fn broker_committed_offset(group_id: &str, topic: &str, partition: i32) -> Option<i64> {let probe: BaseConsumer = ClientConfig::new().set("bootstrap.servers", KAFKA_BROKERS)@@ -1250,8 +1312,6 @@ async fn wait_until(timeout: Duration, msg: &str, mut cond: impl FnMut() -> bool}}-/// Index of the single worker that has received a sub-batch so far.-/// Panics if zero or more than one have — callers rely on a unique pin.fn sole_arrived_worker(harness: &Harness) -> usize {let arrived: Vec<usize> = harness.workers@@ -1268,14 +1328,6 @@ fn sole_arrived_worker(harness: &Harness) -> usize {arrived[0]}-// ── Tests ──────────────────────────────────────────────────────────────────--/// Kafka partition order is preserved per distinct_id on each worker.-///-/// A key is placed on a worker as each of its requests is sent, so the same-/// distinct_id can land on different workers across requests. The guarantee-/// is narrower: wherever a distinct_id's messages land, they arrive in Kafka-/// sequence order on that worker.#[rstest]#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]@@ -1302,12 +1354,10 @@ async fn messages_per_distinct_id_arrive_in_order(#[case] pack_targets: PackTargharness.wait_for(24, Duration::from_secs(15)).await;- // All messages must arrive.let total: usize = harness.workers.iter().map(|w| w.count()).sum();assert_eq!(total, 24, "expected 24 messages total, got {total}");- // On whichever worker(s) a distinct_id's messages land, they must be in- // ascending sequence order — no reordering within a single worker's delivery.+ // A key can move between workers across requests, so order holds per worker.for worker in &harness.workers {for user in ["user-1", "user-2", "user-3"] {let seqs = worker.seqs_for(user);@@ -1321,10 +1371,6 @@ async fn messages_per_distinct_id_arrive_in_order(#[case] pack_targets: PackTargharness.stop().await;}-/// A partial ack redelivers only the messages the worker left unfinished. One-/// Kafka key carries two distinct ids, as a cookieless `token:ip` key does, and-/// the worker leaves one distinct id's messages unfinished while it finishes-/// the other's, so the unfinished messages are not the last of their key.#[tokio::test]async fn a_partial_ack_redelivers_only_the_unfinished_messages() {let topic = format!("e2e-partial-{}", Uuid::new_v4());@@ -1363,9 +1409,6 @@ async fn a_partial_ack_redelivers_only_the_unfinished_messages() {harness.stop().await;}-/// A consumer crash after a partial ack, while its resend is unacked, loses-/// none of the unfinished messages. The partial ack completed the poll's other-/// messages, so a commit past an unfinished offset would skip it on restart.#[tokio::test]async fn a_crash_before_a_partial_acks_resend_loses_nothing() {let topic = format!("e2e-partial-crash-{}", Uuid::new_v4());@@ -1393,8 +1436,7 @@ async fn a_crash_before_a_partial_acks_resend_loses_nothing() {"visitor A's messages are unfinished");- // The consumer commits asynchronously, so a wrong commit can land after- // the crash and go unseen by the restart.+ // Commits are async: wait so a wrong commit lands before the crash.tokio::time::sleep(Duration::from_millis(500)).await;assert_eq!(harness.committed_offset(0),@@ -1424,9 +1466,6 @@ async fn a_crash_before_a_partial_acks_resend_loses_nothing() {harness.stop().await;}-/// A partial ack that arrives after its partition was revoked resends nothing.-/// The new owner replays the partition from its committed offset, so a resend-/// by the old owner would deliver the unfinished messages twice.#[tokio::test]async fn a_partial_ack_after_a_revoke_resends_nothing() {let topic = format!("e2e-partial-revoke-{}", Uuid::new_v4());@@ -1441,18 +1480,14 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {.chain(alternating_visitors(1, "tok:203.0.113.8", 10)).collect();- // The consumer stops polling Kafka after an empty poll while a poll is in- // flight, and only a polling consumer takes part in a rebalance, so a- // trickle keeps it polling. A cold producer's first send can miss the- // first 100 ms poll window, so the producer connects first.+ // A trickle keeps the consumer polling, so it takes part in the rebalance.let producer = make_producer();producer.client().fetch_metadata(Some(&topic), Duration::from_secs(5)).expect("trickle producer metadata");- // Hold the first request in flight across the rebalance. The worker stream- // keeps one request unacked, so the trickle's requests wait behind it.+ // Hold the first request; the trickle's requests queue behind it (one un-acked per worker).let gate = harness.workers[0].block().await;produce_keyed_in_one_transaction(&topic, "tok", &records).await;let trickling = Arc::new(AtomicBool::new(true));@@ -1481,9 +1516,7 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {let second = harness.join_second_consumer().await;- // An eager rebalance revokes both of the first consumer's partitions. The- // held request fences at the transport's 10 s ack timeout and comes back- // as a replay, so the rebalance must land first.+ // Wait under the 10 s ack timeout, after which the held request returns as a replay.wait_until(Duration::from_secs(8),"the rebalance to revoke both partitions",@@ -1516,8 +1549,7 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {},).await;- // The new owner delivers everything on its own, so the old owner may not- // have settled the partial ack yet. A wrong resend would be in flight.+ // The old owner may not have settled the partial ack yet; wait for it.wait_until(Duration::from_secs(15),"the old owner to settle the partial ack",@@ -1542,9 +1574,6 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {harness.stop().await;}-/// A message that never fits the budget still lands. Its budgeted resend-/// finishes nothing, so the next goes out without a budget, and the partition-/// commits.#[tokio::test]async fn a_message_that_never_fits_the_budget_lands_without_one() {let topic = format!("e2e-partial-escalate-{}", Uuid::new_v4());@@ -1577,18 +1606,13 @@ async fn a_message_that_never_fits_the_budget_lands_without_one() {harness.stop().await;}-/// The byte bound ends collection before the count bound when payloads are-/// large, so batch memory tracks bytes rather than event count.-///-/// Twelve 2KiB messages sit far inside the 50-message count cap, so without the-/// byte bound they batch together; the 4KiB cap admits at most two per batch-/// (the second crosses it). The count cap alone cannot express that, which is-/// the regression: a lane whose events are large gets the memory of a full-/// count batch no matter how the count is tuned.+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn byte_bound_caps_batches_below_the_count_bound() {+async fn byte_bound_caps_batches_below_the_count_bound(#[case] polling: Polling) {let topic = format!("e2e-bytecap-{}", Uuid::new_v4());- let harness = Harness::start_byte_capped(&topic, 4096).await;+ let harness = Harness::start_byte_capped(&topic, 4096, polling).await;let producer = make_producer();let payload = vec![b'x'; 2048];@@ -1606,32 +1630,28 @@ async fn byte_bound_caps_batches_below_the_count_bound() {harness.wait_for(12, Duration::from_secs(20)).await;- let sizes = harness.workers[0].batch_sizes();+ let sizes = harness.poll_sizes();assert_eq!(sizes.iter().sum::<usize>(),12,- "every message must still be delivered, got batches {sizes:?}"+ "every message must still be polled, got polls {sizes:?}");- // A collection timeout can only make a batch smaller, so this holds- // regardless of how production interleaves with the 100ms window.+ // A batch timeout only shrinks polls, so producer timing cannot break this.assert!(sizes.iter().all(|&n| n <= 2),- "no batch may exceed the byte bound by more than one message, got {sizes:?}"+ "no poll may exceed the byte bound by more than one message, got {sizes:?}");harness.stop().await;}-/// A single message larger than the whole byte bound still moves.-///-/// The bound is checked before appending, so the first message of a batch is-/// always admitted. Checking after would let one oversized payload produce an-/// empty batch and wedge its partition — the message can never be skipped, and-/// no smaller batch can ever contain it.+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn message_larger_than_byte_bound_is_still_delivered() {+async fn message_larger_than_byte_bound_is_still_delivered(#[case] polling: Polling) {let topic = format!("e2e-bytecap-oversize-{}", Uuid::new_v4());- let harness = Harness::start_byte_capped(&topic, 4096).await;+ let harness = Harness::start_byte_capped(&topic, 4096, polling).await;let producer = make_producer();produce_raw(@@ -1655,8 +1675,6 @@ async fn message_larger_than_byte_bound_is_still_delivered() {harness.stop().await;}-/// When a worker becomes unhealthy, its sticky pins are dropped and subsequent-/// messages for the same key are rerouted to a healthy worker.#[tokio::test]async fn failing_worker_triggers_rerouting() {let topic = format!("e2e-failover-{}", Uuid::new_v4());@@ -1672,14 +1690,12 @@ async fn failing_worker_triggers_rerouting() {let producer = make_producer();- // Batch 1: 4 messages for user-1 (partition 0) and user-2 (partition 1).for seq in 0..4usize {produce(&producer, &topic, 0, "tok", "user-1", seq).await;produce(&producer, &topic, 1, "tok", "user-2", seq).await;}harness.wait_for(8, Duration::from_secs(10)).await;- // Identify which worker owns user-1 and take it down.let dead_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {0} else {@@ -1691,17 +1707,13 @@ async fn failing_worker_triggers_rerouting() {.healthy.store(false, Ordering::Relaxed);- // Wait for probe failures → Unhealthy → dead declaration.- // probe_interval*probe_failure_threshold + dead_declaration ≈ 300ms; use 600ms to be safe.tokio::time::sleep(Duration::from_millis(600)).await;- // Batch 2: 4 more messages for user-1. The pin was dropped; should route to the live worker.for seq in 4..8usize {produce(&producer, &topic, 0, "tok", "user-1", seq).await;}harness.wait_for(12, Duration::from_secs(10)).await;- // Batch-2 messages (seq 4–7) must all be on the live worker and in order.let live_seqs = harness.workers[live_idx].seqs_for("user-1");let batch2_seqs: Vec<usize> = live_seqs.iter().copied().filter(|&s| s >= 4).collect();assert_eq!(@@ -1713,9 +1725,6 @@ async fn failing_worker_triggers_rerouting() {harness.stop().await;}-/// A worker whose sends fail (5xx through every retry) is a fault for passive-/// health, not backpressure: the failed sends must degrade it even though the-/// readiness probe keeps passing.#[tokio::test]async fn failed_sends_degrade_passive_health() {let topic = format!("e2e-passive-fault-{}", Uuid::new_v4());@@ -1745,11 +1754,6 @@ async fn failed_sends_degrade_passive_health() {harness.stop().await;}-/// Graceful drain end-to-end: a worker that begins draining receives no new-/// work, keeps the messages it already has (no reprocessing, no loss), and-/// subsequent messages for the same distinct_id reroute in order to a surviving-/// worker. The drainer stays alive throughout (it is not failed), so the probe-/// must not declare it dead.#[tokio::test]async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {let topic = format!("e2e-drain-{}", Uuid::new_v4());@@ -1765,14 +1769,12 @@ async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {let producer = make_producer();- // Batch 1: 4 messages each for user-1 (partition 0) and user-2 (partition 1).for seq in 0..4usize {produce(&producer, &topic, 0, "tok", "user-1", seq).await;produce(&producer, &topic, 1, "tok", "user-2", seq).await;}harness.wait_for(8, Duration::from_secs(10)).await;- // Identify which worker owns user-1 and begin draining it.let drain_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {0} else {@@ -1782,9 +1784,6 @@ async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {let drain_url = harness.workers[drain_idx].url.clone();harness.registry.start_draining(&drain_url);- // With the reaper running (as in production), an idle drainer is promptly- // reaped — cleanly removed from the pool once its in-flight hits zero —- // rather than lingering or being probed to death mid-drain.wait_until(Duration::from_secs(2), "idle drainer to be reaped", || {!harness.registry@@ -1794,14 +1793,11 @@ async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {}).await;- // Batch 2: 4 more messages for user-1. The drainer takes no new work, so- // these must route to the live worker, in order.for seq in 4..8usize {produce(&producer, &topic, 0, "tok", "user-1", seq).await;}harness.wait_for(12, Duration::from_secs(10)).await;- // No reprocessing / no loss: each user-1 seq 0..8 is delivered exactly once.let mut all_user1: Vec<usize> = harness.workers.iter()@@ -1814,14 +1810,12 @@ async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {"user-1 messages were lost or duplicated: {all_user1:?}");- // The draining worker received no new work after draining began.let drained_seqs = harness.workers[drain_idx].seqs_for("user-1");assert!(drained_seqs.iter().all(|&s| s < 4),"draining worker received new work: {drained_seqs:?}");- // Batch-2 messages all landed on the live worker, in ascending order.let live_batch2: Vec<usize> = harness.workers[live_idx].seqs_for("user-1").into_iter()@@ -1836,13 +1830,6 @@ async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {harness.stop().await;}-// ── Failure-triggered defer + flush at the consumer level ────────────────────-//-// These drive the batcher's requeue-and-replay loop end-to-end through real-// Kafka. A "blocking" worker holds a request in flight so a test can flip it-// to a failure before it responds; the failed send then exercises the-// batcher's requeue and its retry wakeup.-/// Whether `url` is currently routable (healthy or degraded) per the registry.fn in_pool(harness: &Harness, url: &str) -> bool {harness@@ -1852,8 +1839,6 @@ fn in_pool(harness: &Harness, url: &str) -> bool {.any(|w| w.as_ref() == url)}-/// A send that fails mid-flight is deferred and replayed, in order, to the-/// surviving worker — no loss, no duplication, no reordering.#[tokio::test]async fn send_failure_mid_flight_replays_to_survivor_in_order() {let topic = format!("e2e-replay-single-{}", Uuid::new_v4());@@ -1884,7 +1869,6 @@ async fn send_failure_mid_flight_replays_to_survivor_in_order() {let failed = sole_arrived_worker(&harness);let survivor = 1 - failed;- // Turn the held request into a failure, then release everything.harness.workers[failed].healthy.store(false, Ordering::SeqCst);@@ -1907,13 +1891,9 @@ async fn send_failure_mid_flight_replays_to_survivor_in_order() {harness.stop().await;}-/// In a batch split across two workers, only the failed sub-batch is deferred-/// and replayed; the successful one is not re-sent. Everything ends up on the-/// survivor, in order, with the failed worker having recorded nothing.#[tokio::test]async fn partial_send_failure_replays_only_the_failed_subbatch() {let topic = format!("e2e-replay-partial-{}", Uuid::new_v4());- // Each four-message key fills its own request, so there are two sub-batches.let one_key_of_four_per_request = PackTargets {events: NonZeroUsize::new(4),bytes: None,@@ -1935,10 +1915,7 @@ async fn partial_send_failure_replays_only_the_failed_subbatch() {guards.push(Some(w.block().await));}- // Two keys over two workers → P2C places one per worker: one batch, two- // sub-batches, one per worker. The keys must share one Kafka batch: at- // max_in_flight=1 a Kafka batch with only one key blocks the next batch,- // so the second worker would never receive a sub-batch.+ // One poll must carry both keys; at max_in_flight=1 a one-key poll would block the second.let records: Vec<(&str, usize)> = (0..4usize).flat_map(|seq| [("user-1", seq), ("user-2", seq)]).collect();@@ -1950,8 +1927,6 @@ async fn partial_send_failure_replays_only_the_failed_subbatch() {).await;- // Fail worker 0's sub-batch only. Whatever key it held replays to worker 1,- // so worker 1 ends with both keys and worker 0 with nothing.harness.workers[0].healthy.store(false, Ordering::SeqCst);guards[1] = None;guards[0] = None;@@ -1977,11 +1952,6 @@ async fn partial_send_failure_replays_only_the_failed_subbatch() {harness.stop().await;}-/// When a send fails and there is no healthy worker to replay to, the deferred-/// work is held (not lost, not dropped) and the flush loop retries until a-/// worker returns, then drains in order. Waiting for a worker, and waiting on-/// a slow one once found, both outlast the liveness deadline here: neither is-/// a stall, so the lifecycle monitor must not shut the consumer down.#[tokio::test]async fn deferred_flush_retries_until_a_worker_recovers() {let topic = format!("e2e-replay-wait-{}", Uuid::new_v4());@@ -2013,11 +1983,7 @@ async fn deferred_flush_retries_until_a_worker_recovers() {harness.workers[0].healthy.store(false, Ordering::SeqCst);drop(guard0);- // Wait until worker 0 has also left the pool. The failed send is now- // deferred with nowhere to go. (At max_in_flight=1 the consumer is parked in- // the flush loop for the oldest batch and stops consuming, so only that- // batch's messages are stashed — Kafka polling may have split the produced- // records across batches; the exact count doesn't matter.)+ // Only the oldest poll is stashed, so read its count rather than assert 4.wait_until(Duration::from_secs(10),"worker 0 to leave the pool",@@ -2031,8 +1997,6 @@ async fn deferred_flush_retries_until_a_worker_recovers() {).await;- // The deferred work is held steady — the flush loop is backing off, not- // dropping anything — for as long as no worker is available.let held = harness.observer.pending_messages();tokio::time::sleep(liveness_deadline * 4).await;assert_eq!(@@ -2045,8 +2009,7 @@ async fn deferred_flush_retries_until_a_worker_recovers() {"waiting for a worker must keep the liveness heartbeat going");- // Recover worker 1 while it is blocked → the flush loop sends the stash to- // it and waits on the in-flight request past the liveness deadline.+ // Worker 1 rejoins blocked, so the flush waits on it past the liveness deadline.let guard1 = harness.workers[1].block().await;harness.workers[1].healthy.store(true, Ordering::SeqCst);wait_until(Duration::from_secs(10), "replay to reach worker 1", || {@@ -2059,8 +2022,6 @@ async fn deferred_flush_retries_until_a_worker_recovers() {"waiting on a slow worker must keep the liveness heartbeat going");- // Release worker 1 → the stash drains, then the consumer resumes and- // delivers the rest. All of user-1 lands on worker 1, in order.drop(guard1);harness.wait_for(4, Duration::from_secs(15)).await;@@ -2078,9 +2039,6 @@ async fn deferred_flush_retries_until_a_worker_recovers() {harness.stop().await;}-/// With no worker ever coming back, the batch must still fail: the flush-/// loop's own `deferred_flush_timeout` ends it, not a liveness stall, so the-/// exit lands at the flush timeout rather than at the (shorter) stall window.#[tokio::test]async fn no_worker_exit_comes_from_the_flush_timeout_not_a_liveness_stall() {let topic = format!("e2e-flush-timeout-liveness-{}", Uuid::new_v4());@@ -2135,10 +2093,6 @@ async fn no_worker_exit_comes_from_the_flush_timeout_not_a_liveness_stall() {);}-/// A drain that is slow but moving must complete even when it outlasts both the-/// liveness deadline and the flush timeout: every landing resets the flush-/// deadline, and the wait in between keeps the heartbeat going. Two keys land-/// in two flush rounds, the first only after the original deadline has passed.#[tokio::test]async fn slow_deferred_drain_with_progress_outlasts_the_flush_timeout() {let topic = format!("e2e-slow-drain-{}", Uuid::new_v4());@@ -2172,8 +2126,7 @@ async fn slow_deferred_drain_with_progress_outlasts_the_flush_timeout() {}).await;- // Workers 1 and 2 rejoin, blocked, before worker 0 fails, so the flush- // sees both and spreads the two keys over them, one request each.+ // Rejoin workers 1 and 2 before worker 0 fails, so the flush spreads the keys over them.let mut guard1 = Some(harness.workers[1].block().await);let guard2 = harness.workers[2].block().await;for i in [1, 2] {@@ -2258,8 +2211,6 @@ async fn slow_deferred_drain_with_progress_outlasts_the_flush_timeout() {harness.stop().await;}-/// A failed multi-key sub-batch replays every key, each in its own Kafka order,-/// once a worker is available.#[rstest]#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]@@ -2278,7 +2229,7 @@ async fn multi_key_send_failure_replays_every_key_in_order(#[case] pack_targets:.await;let producer = make_producer();- // Worker 1 out so all three keys land on worker 0 as one sub-batch.+ // Worker 1 out so all three keys land on worker 0.harness.workers[1].healthy.store(false, Ordering::SeqCst);wait_until(Duration::from_secs(10),@@ -2298,7 +2249,6 @@ async fn multi_key_send_failure_replays_every_key_in_order(#[case] pack_targets:}).await;- // Fail worker 0 and bring worker 1 back as the replay target.harness.workers[0].healthy.store(false, Ordering::SeqCst);harness.workers[1].healthy.store(true, Ordering::SeqCst);wait_until(@@ -2327,8 +2277,6 @@ async fn multi_key_send_failure_replays_every_key_in_order(#[case] pack_targets:harness.stop().await;}-/// A replay whose target also fails is re-deferred and retried until it lands —-/// cascading failures during flush never lose or reorder the deferred work.#[tokio::test]async fn flush_target_failure_re_defers_then_replays_in_order() {let topic = format!("e2e-replay-cascade-{}", Uuid::new_v4());@@ -2358,8 +2306,7 @@ async fn flush_target_failure_re_defers_then_replays_in_order() {let first = sole_arrived_worker(&harness);let second = 1 - first;- // Fail the initial send → the batch defers and the flush re-routes to the- // other worker, whose request we are holding via its gate.+ // Fail the first send; the replay goes to the other, blocked worker.harness.workers[first].healthy.store(false, Ordering::SeqCst);@@ -2385,9 +2332,7 @@ async fn flush_target_failure_re_defers_then_replays_in_order() {},).await;- // The failed replay may still be parked on its worker stream (reconnect- // backoff) when the pool empties; it lands in the stash once that send- // fails too, and stays there with nowhere to route.+ // The failed replay can wait in reconnect backoff before it reaches the stash.wait_until(Duration::from_secs(10),"re-deferred work to be held after the replay target also fails",@@ -2395,7 +2340,6 @@ async fn flush_target_failure_re_defers_then_replays_in_order() {).await;- // Recover the second worker → the replay finally lands, in order.harness.workers[second].healthy.store(true, Ordering::SeqCst);@@ -2415,12 +2359,6 @@ async fn flush_target_failure_re_defers_then_replays_in_order() {harness.stop().await;}-// ── Pool-wide outages, forced reap, crash restart, ambiguous ACKs ────────────--/// A transient full-pool drain — every worker leaving at once, as during a-/// deploy overlap — must not fail the batch: fresh keys that can't route-/// anywhere are held and delivered once a worker returns, instead of being-/// dropped (which fails the batch and restarts the process).#[tokio::test]async fn full_pool_drain_holds_fresh_keys_until_a_worker_returns() {let topic = format!("e2e-full-drain-{}", Uuid::new_v4());@@ -2452,7 +2390,6 @@ async fn full_pool_drain_holds_fresh_keys_until_a_worker_returns() {"nothing can be delivered while the whole pool is drained");- // One worker returns — the held messages must deliver to it, in order.harness.registry.add_worker(WorkerId::from(harness.workers[0].url.as_str()));@@ -2466,11 +2403,6 @@ async fn full_pool_drain_holds_fresh_keys_until_a_worker_returns() {harness.stop().await;}-/// A worker that leaves the pool but never finishes its in-flight work is-/// force-reaped at the drain timeout — removed from the registry and transport-/// while its request is still live. The reap must not break the in-flight-/// resolve, and the key's deferred messages must re-route to a survivor in-/// order once the held batch completes.#[tokio::test]async fn forced_reap_at_drain_timeout_reroutes_deferred_work() {let topic = format!("e2e-forced-reap-{}", Uuid::new_v4());@@ -2510,8 +2442,6 @@ async fn forced_reap_at_drain_timeout_reroutes_deferred_work() {}).await;- // The drain timeout passes with the batch still held → the reaper force-- // removes the worker while its request is live.wait_until(Duration::from_secs(10),"the drainer to be force-reaped",@@ -2525,8 +2455,7 @@ async fn forced_reap_at_drain_timeout_reroutes_deferred_work() {).await;- // Release everything: batch 1 completes on the (already removed) worker,- // then batch 2 flushes to the survivor.+ // Release all: batch 1 completes on the reaped worker, batch 2 flushes to the survivor.guards.clear();harness.wait_for(8, Duration::from_secs(15)).await;@@ -2549,10 +2478,6 @@ async fn forced_reap_at_drain_timeout_reroutes_deferred_work() {harness.stop().await;}-/// A consumer crash after a batch reached a worker but before its offsets were-/// committed must not lose anything: the restarted consumer resumes from the-/// last commit and redelivers. Duplicates are allowed (at-least-once); loss is-/// not.#[tokio::test]async fn consumer_crash_before_commit_redelivers_without_loss() {let topic = format!("e2e-restart-{}", Uuid::new_v4());@@ -2567,8 +2492,7 @@ async fn consumer_crash_before_commit_redelivers_without_loss() {.await;let producer = make_producer();- // Hold both workers so the first batch is in flight but never ACKs — its- // offsets are never committed.+ // Hold both workers so the first batch never acks or commits.let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();for w in &harness.workers {guards.push(Some(w.block().await));@@ -2582,9 +2506,7 @@ async fn consumer_crash_before_commit_redelivers_without_loss() {}).await;- // Crash the consumer mid-flight, then release the held requests. Whether- // the worker ends up recording that first delivery is exactly the ambiguity- // a restart must tolerate: duplicates allowed, loss not.+ // The worker may record the first delivery after the crash: duplicates allowed, loss not.harness.crash_consumer();guards.clear();@@ -2622,8 +2544,6 @@ async fn assert_commits_track_the_ledger_frontier(mut harness: Harness) {).await;- // A clean frontier is a usable resume point: after a crash, only the new- // messages arrive.harness.crash_consumer();harness.restart_consumer(fast_registry_config()).await;for seq in 10..15usize {@@ -2845,13 +2765,23 @@ async fn group_granularity_commits_a_partition_while_another_stalls() {harness.stop().await;}+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn a_partition_at_its_budget_pauses_until_a_commit_frees_it() {+async fn a_partition_at_its_budget_pauses_until_a_commit_frees_it(#[case] polling: Polling) {let topic = format!("e2e-partition-budget-{}", Uuid::new_v4());- let budget = BudgetLimits::new(NonZeroU64::new(5), None, 0.5).expect("a valid budget");- let harness =- Harness::start_with_partition_budget(common::PRODUCTION_PACK_TARGETS, &topic, 1, 1, budget)- .await;+ let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)+ .expect("a valid budget");+ let harness = Harness::start_with_partition_budget(+ common::PRODUCTION_PACK_TARGETS,+ &topic,+ 1,+ 1,+ budget,+ polling,+ )+ .await;let p0 = TopicPartition::new(&topic, 0);let guard = harness.workers[0].block().await;let producer = make_producer();@@ -2885,14 +2815,24 @@ async fn a_partition_at_its_budget_pauses_until_a_commit_frees_it() {harness.stop().await;}+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn a_partition_at_its_budget_does_not_pause_another() {+async fn a_partition_at_its_budget_does_not_pause_another(#[case] polling: Polling) {let topic = format!("e2e-partition-budget-two-{}", Uuid::new_v4());- let budget = BudgetLimits::new(NonZeroU64::new(5), None, 0.5).expect("a valid budget");+ let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)+ .expect("a valid budget");// One message per request, or the packer puts both keys on one worker.- let harness =- Harness::start_with_partition_budget(common::ONE_MESSAGE_PER_REQUEST, &topic, 2, 2, budget)- .await;+ let harness = Harness::start_with_partition_budget(+ common::ONE_MESSAGE_PER_REQUEST,+ &topic,+ 2,+ 2,+ budget,+ polling,+ )+ .await;let mut guards = stall_two_partitions_on_two_workers(&harness, &topic).await;guards[0] = None;@@ -2936,10 +2876,138 @@ async fn a_partition_at_its_budget_does_not_pause_another() {harness.stop().await;}+#[tokio::test]+async fn continuous_polling_commits_the_ledger_frontier() {+ let topic = format!("e2e-continuous-commit-{}", Uuid::new_v4());+ assert_commits_track_the_ledger_frontier(+ Harness::start_continuous(+ common::PRODUCTION_PACK_TARGETS,+ &topic,+ 1,+ 1,+ BudgetLimits::new(+ NonZeroU64::new(1_000),+ NonZeroU64::new(UNBOUNDED_BYTES),+ 0.8,+ )+ .expect("a valid budget"),+ ChargeLimit::new(NonZeroU64::new(1_000), NonZeroU64::new(UNBOUNDED_BYTES)),+ )+ .await,+ )+ .await;+}++#[tokio::test]+async fn the_in_flight_limit_stops_reading_until_a_commit_frees_room() {+ let topic = format!("e2e-in-flight-limit-{}", Uuid::new_v4());+ let harness = Harness::start_continuous(+ common::PRODUCTION_PACK_TARGETS,+ &topic,+ 1,+ 1,+ BudgetLimits::new(+ NonZeroU64::new(1_000),+ NonZeroU64::new(UNBOUNDED_BYTES),+ 0.8,+ )+ .expect("a valid budget"),+ ChargeLimit::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES)),+ )+ .await;+ let p0 = TopicPartition::new(&topic, 0);+ let guard = harness.workers[0].block().await;+ let backlog: Vec<(&str, usize)> = (0..120).map(|seq| ("user-a", seq)).collect();++ produce_in_one_transaction(&topic, 0, "tok", &backlog).await;+ wait_until(+ Duration::from_secs(10),+ "the in-flight limit to fill",+ || harness.ledger.held(&p0).offsets >= 5,+ )+ .await;+ tokio::time::sleep(Duration::from_secs(1)).await;+ assert_eq!(+ harness.ledger.held(&p0).offsets,+ 50,+ "a ready backlog stops one poll of the batch size past the limit"+ );++ drop(guard);+ wait_until(+ Duration::from_secs(10),+ "reading to resume and drain",+ || harness.workers[0].count() == 120,+ )+ .await;++ harness.stop().await;+}++/// On this single-threaded runtime the batcher runs only when the consumer+/// loop yields. A ready backlog keeps `recv` ready on every turn, so the first+/// request must reach the worker before the loop has charged the whole backlog.+#[tokio::test]+async fn a_ready_backlog_does_not_keep_the_batcher_from_running() {+ const BACKLOG: usize = 2_000;+ let topic = format!("e2e-continuous-backlog-{}", Uuid::new_v4());+ let harness = Harness::start_continuous(+ common::PRODUCTION_PACK_TARGETS,+ &topic,+ 1,+ 1,+ BudgetLimits::new(+ NonZeroU64::new(100_000),+ NonZeroU64::new(UNBOUNDED_BYTES),+ 0.8,+ )+ .expect("a valid budget"),+ ChargeLimit::new(NonZeroU64::new(100_000), NonZeroU64::new(UNBOUNDED_BYTES)),+ )+ .await;+ let p0 = TopicPartition::new(&topic, 0);+ let held_at_arrival: Arc<Mutex<Vec<usize>>> = Arc::new(Mutex::new(Vec::new()));+ {+ let ledger = Arc::clone(&harness.ledger);+ let p0 = p0.clone();+ let held_at_arrival = Arc::clone(&held_at_arrival);+ harness.workers[0].on_arrival(move || {+ held_at_arrival+ .lock()+ .unwrap()+ .push(ledger.held(&p0).offsets);+ });+ }+ let guard = harness.workers[0].block().await;+ let backlog: Vec<(&str, usize)> = (0..BACKLOG).map(|seq| ("user-a", seq)).collect();++ produce_in_one_transaction(&topic, 0, "tok", &backlog).await;+ wait_until(+ Duration::from_secs(10),+ "the first request to reach the worker",+ || harness.workers[0].arrived_count() >= 1,+ )+ .await;+ let held = held_at_arrival.lock().unwrap()[0];+ assert!(+ held < BACKLOG / 2,+ "the loop charged {held} of {BACKLOG} offsets before the first request reached the worker"+ );++ drop(guard);+ wait_until(Duration::from_secs(10), "the backlog to drain", || {+ harness.workers[0].count() == BACKLOG+ })+ .await;++ harness.stop().await;+}+#[tokio::test]async fn a_partition_paused_at_its_budget_fetches_again_after_a_rebalance_returns_it() {let topic = format!("e2e-partition-budget-rebalance-{}", Uuid::new_v4());- let budget = BudgetLimits::new(NonZeroU64::new(5), None, 0.5).expect("a valid budget");+ let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)+ .expect("a valid budget");let harness = Harness::start_with_partition_budget_committing_every(common::PRODUCTION_PACK_TARGETS,&topic,@@ -3004,10 +3072,6 @@ async fn a_partition_paused_at_its_budget_fetches_again_after_a_rebalance_returnharness.stop().await;}-/// A worker that processes a batch but whose ACK is lost is indistinguishable-/// from a failed send, so the consumer replays the batch: duplicates are the-/// accepted cost of at-least-once, loss never is, and the replay lands behind-/// the original delivery.#[tokio::test]async fn lost_ack_after_processing_replays_without_loss() {let topic = format!("e2e-lost-ack-{}", Uuid::new_v4());@@ -3022,8 +3086,7 @@ async fn lost_ack_after_processing_replays_without_loss() {.await;let producer = make_producer();- // Arm both workers (the batch could land on either): the first sub-batch each- // receives is processed and recorded, but answered with an error.+ // Arm both workers: the batch can land on either.for w in &harness.workers {w.ack_lost_once.store(true, Ordering::SeqCst);}@@ -3064,13 +3127,6 @@ async fn lost_ack_after_processing_replays_without_loss() {harness.stop().await;}-// ── Poison batches and contract violations ───────────────────────────────────--/// A batch every worker permanently rejects with a 4xx must fail the consumer-/// within the flush timeout — failing safe (nothing ingested, nothing-/// committed, redelivered after restart) rather than spinning forever. Until a-/// DLQ path exists, a poison batch crash-loops the process by design; this-/// pins the "safe" half of that trade-off.#[tokio::test]async fn poison_batch_fails_safely_without_committing() {let topic = format!("e2e-poison-{}", Uuid::new_v4());@@ -3106,10 +3162,6 @@ async fn poison_batch_fails_safely_without_committing() {);}-/// A worker that ingests a batch but reports fewer accepted messages than were-/// sent violates the transport contract. The consumer must treat the batch as-/// incomplete and fail it — never commit offsets over an under-acknowledged-/// batch, which would silently lose the unaccounted messages.#[tokio::test]async fn partial_acceptance_fails_the_batch_without_commit() {let topic = format!("e2e-partial-accept-{}", Uuid::new_v4());@@ -3145,11 +3197,6 @@ async fn partial_acceptance_fails_the_batch_without_commit() {);}-// ── Drain edge cases and malformed input ─────────────────────────────────────--/// A drainer that crashes mid-drain (its in-flight send fails instead of-/// finishing) must not wedge the drain: both its failed in-flight batch and-/// the work deferred behind it replay to the survivor, in order.#[tokio::test]async fn drainer_crash_mid_drain_replays_to_survivor_in_order() {let topic = format!("e2e-drain-crash-{}", Uuid::new_v4());@@ -3213,9 +3260,6 @@ async fn drainer_crash_mid_drain_replays_to_survivor_in_order() {harness.stop().await;}-/// The whole pool failing at once with multiple keys in flight, at-/// max_in_flight > 1: everything is stashed while nothing is routable, then-/// drains to the first recovered worker with per-key order intact.#[rstest]#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]@@ -3245,9 +3289,7 @@ async fn full_pool_send_failure_with_multiple_keys_recovers_in_order(produce(&producer, &topic, 0, "tok", "user-1", seq).await;produce(&producer, &topic, 1, "tok", "user-2", seq).await;}- // One message per request splits the keys into two requests, and- // P2C places them one per worker. At the production target- // both keys fit in one request.+ // One message per request puts each key on its own worker; production targets pack both.let split = pack_targets == common::ONE_MESSAGE_PER_REQUEST;wait_until(Duration::from_secs(10), "work to be in flight", || {if split {@@ -3258,8 +3300,6 @@ async fn full_pool_send_failure_with_multiple_keys_recovers_in_order(}).await;- // The whole pool fails at once with work in flight — everything must be- // stashed, not lost, while nothing is routable.for w in &harness.workers {w.healthy.store(false, Ordering::SeqCst);}@@ -3271,7 +3311,6 @@ async fn full_pool_send_failure_with_multiple_keys_recovers_in_order().await;- // One worker recovers → everything drains to it, per-key order intact.harness.workers[1].healthy.store(true, Ordering::SeqCst);harness.wait_for(8, Duration::from_secs(15)).await;@@ -3291,9 +3330,6 @@ async fn full_pool_send_failure_with_multiple_keys_recovers_in_order(harness.stop().await;}-/// Messages without token/distinct_id headers route under synthetic-/// per-message keys; a send failure must replay them exactly once, like any-/// other message.#[tokio::test]async fn headerless_messages_survive_send_failure_replay() {let topic = format!("e2e-headerless-{}", Uuid::new_v4());@@ -3334,7 +3370,6 @@ async fn headerless_messages_survive_send_failure_replay() {}).await;- // Fail worker 0 and recover worker 1 as the replay target.harness.workers[0].healthy.store(false, Ordering::SeqCst);harness.workers[1].healthy.store(true, Ordering::SeqCst);wait_until(@@ -3351,8 +3386,7 @@ async fn headerless_messages_survive_send_failure_replay() {0,"failed worker recorded nothing");- // Headerless messages record under an empty distinct_id; all three must- // replay exactly once (order across distinct synthetic keys is undefined).+ // Headerless messages have an empty distinct_id and distinct keys, so order is free.let mut seqs = harness.workers[1].seqs_for("");seqs.sort_unstable();assert_eq!(@@ -3364,9 +3398,6 @@ async fn headerless_messages_survive_send_failure_replay() {harness.stop().await;}-/// A non-UTF-8 payload cannot round-trip the string wire format — its body is-/// nulled in transit today (a known content-loss gap) — but the event itself-/// must still route via its headers and reach a worker, not be dropped.#[tokio::test]async fn non_utf8_payload_is_delivered_not_lost() {let topic = format!("e2e-non-utf8-{}", Uuid::new_v4());@@ -3390,7 +3421,6 @@ async fn non_utf8_payload_is_delivered_not_lost() {key: "distinct_id",value: Some("user-1"),});- // 0xF0 0x28 0x8C 0x28 is invalid UTF-8.produce_raw(&producer,&topic,@@ -3411,11 +3441,6 @@ async fn non_utf8_payload_is_delivered_not_lost() {harness.stop().await;}-// ── Fleet redeploy and consumer-group membership ─────────────────────────────--/// A full fleet replacement, reconciled exactly as EndpointSlice discovery-/// does: the desired set flips to brand-new workers, the old ones drain and-/// are reaped, and traffic keeps flowing — exactly once, in per-key order.#[tokio::test]async fn fleet_redeploy_migrates_work_to_new_workers_without_loss() {let topic = format!("e2e-redeploy-{}", Uuid::new_v4());@@ -3456,13 +3481,11 @@ async fn fleet_redeploy_migrates_work_to_new_workers_without_loss() {}harness.wait_for(40, Duration::from_secs(20)).await;- // The old fleet is drained and reaped out of the pool entirely.wait_until(Duration::from_secs(5), "old workers to be reaped", || {harness.registry.worker_count() == 2}).await;- // Exactly once, in per-key order, across the redeploy.let log = harness.delivery_log.lock().unwrap().clone();assert_eq!(log.len(), 40, "exactly-once across the redeploy");for user in ["user-1", "user-2"] {@@ -3486,26 +3509,16 @@ async fn fleet_redeploy_migrates_work_to_new_workers_without_loss() {harness.stop().await;}-/// A second consumer joining the same group rebalances partitions away from-/// the first while it holds an uncommitted in-flight batch. Redelivery of that-/// batch on the new owner may duplicate (at-least-once) — but nothing may be-/// lost across the handover.+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn second_consumer_joining_the_group_preserves_all_messages() {+async fn second_consumer_joining_the_group_preserves_all_messages(#[case] polling: Polling) {let topic = format!("e2e-two-consumers-{}", Uuid::new_v4());- let harness = Harness::start(- &topic,- 2,- 2,- 1,- Duration::from_secs(60),- fast_registry_config(),- )- .await;+ let harness = Harness::start_polling(&topic, 2, 2, 1, 0, polling).await;let producer = make_producer();- // The first consumer owns both partitions and has an uncommitted batch- // held in flight.+ // Hold an uncommitted batch while the first consumer owns both partitions.let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();for w in &harness.workers {guards.push(Some(w.block().await));@@ -3521,8 +3534,7 @@ async fn second_consumer_joining_the_group_preserves_all_messages() {let second = harness.join_second_consumer().await;- // Release the held batch so the first consumer resumes polling and the- // rebalance can complete, then keep traffic flowing on both partitions.+ // Release the batch so the first consumer polls again and the rebalance completes.guards.clear();tokio::time::sleep(Duration::from_secs(2)).await;for seq in 5..10usize {@@ -3550,28 +3562,14 @@ async fn second_consumer_joining_the_group_preserves_all_messages() {harness.stop().await;}-/// A partition leaves for a second consumer while a batch is in flight, and-/// returns when that consumer shuts down. The first consumer keeps polling-/// under the held batch, so the rebalance lands before the batch settles:-/// its offsets are stamped under generations the revoke replaced, and they-/// drop as stale instead of landing on the new assignment. The returned-/// partition then starts another generation. A ledger bug here would-/// surface as ledger errors or a wedged loop, so continued delivery after-/// both handovers is the health signal.+#[rstest]+#[case::capped(Polling::Capped)]+#[case::continuous(Polling::Continuous)]#[tokio::test]-async fn partition_lost_and_regained_keeps_the_consumer_alive() {+async fn partition_lost_and_regained_keeps_the_consumer_alive(#[case] polling: Polling) {let topic = format!("e2e-regain-{}", Uuid::new_v4());- // Room for many in-flight batches, so the first consumer keeps polling- // while its batches are held and the rebalance can land underneath them.- let harness = Harness::start(- &topic,- 2,- 2,- 128,- Duration::from_secs(60),- fast_registry_config(),- )- .await;+ // 128 in-flight polls keep the consumer polling under held batches.+ let harness = Harness::start_polling(&topic, 2, 2, 128, 0, polling).await;let producer = make_producer();let partitions = [TopicPartition::new(topic.clone(), 0),@@ -3599,10 +3597,7 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {// A second consumer joins and takes a partition away.let second = harness.join_second_consumer().await;- // The first consumer polls only while its polls return messages, and- // only a polling consumer takes part in a rebalance: a trickle holds it- // in the poll loop until the rebalance has reached its ledger. Every- // trickle batch is held like the first one.+ // A trickle keeps the consumer polling until the rebalance reaches its ledger.let rebalanced = || {partitions.iter()@@ -3621,8 +3616,7 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {tokio::time::sleep(Duration::from_millis(20)).await;}- // Release the held batch: its offsets are stamped under generations the- // rebalance replaced, so they settle as stale.+ // The released offsets carry replaced generations, so they settle as stale.guards.clear();for seq in 5..10usize {produce(&producer, &topic, 0, "tok", "user-1", seq).await;@@ -3644,8 +3638,7 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {).await;- // The second consumer leaves; its partition returns to the first under a- // new generation and must consume as a fresh assignment.+ // The partition returns under a new generation.second.stop().await;for seq in 10..15usize {@@ -3671,10 +3664,6 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {harness.stop().await;}-/// A same-name pod restart racing its old instance: a second member joins with-/// the same `group.instance.id`, so the broker fences the first. The fenced-/// instance's client is permanently dead — the consumer must exit (so the pod-/// restarts) rather than keep polling it while reporting healthy.#[tokio::test]async fn fenced_static_member_exits_on_fatal_error() {let topic = format!("e2e-fenced-{}", Uuid::new_v4());@@ -3717,6 +3706,8 @@ async fn fenced_static_member_exits_on_fatal_error() {commit_interval: Duration::ZERO,ledger: Arc::new(TopicOffsetLedger::new()),partition_budget: BudgetLimits::unset(),+ polling: Polling::Capped,+ in_flight_limit: ChargeLimit::unset(),},handle,);@@ -3734,8 +3725,7 @@ async fn fenced_static_member_exits_on_fatal_error() {let usurper = make_kafka_consumer(&topic, &group, Some("pod-1"));let usurper_task = tokio::spawn(async move {let mut stream = usurper.stream();- // A transient connect error is an item too; stopping at it would drop- // the usurper before it joins.+ // Errors are items too; read until the timeout so the usurper joins.let _ = tokio::time::timeout(Duration::from_secs(30), async {while stream.next().await.is_some() {}})@@ -3754,23 +3744,6 @@ async fn fenced_static_member_exits_on_fatal_error() {probe.cancel();}-// ── max_in_flight > 1 churn / invariant suite ────────────────────────────────-//-// At max_in_flight > 1 the consumer processes batches concurrently, so exact-// scenarios can't be scripted deterministically. These tests instead run a high-// volume across many distinct_ids while aggressively churning the worker pool,-// and assert the correctness INVARIANTS on the cross-worker delivery log:-// - exactly-once: every produced (distinct_id, seq) is delivered once — no-// loss, no duplication;-// - ordering: under graceful drain, each distinct_id's messages are delivered-// in Kafka order despite rerouting (a regression of the deferred-replay-// ordering shows up here as an out-of-order seq).-//-// Drain churn (the graceful-drain feature) preserves order, so it is asserted-// strictly. Send-failure churn falls back to at-least-once replay, which can-// interleave a failed-then-replayed message with a later same-key success on a-// flapping worker; those variants assert exactly-once but not strict order.-#[derive(Clone, Copy, Debug)]enum Churn {Drain,@@ -3785,10 +3758,7 @@ async fn sleep_or_cancel(token: &CancellationToken, ms: u64) -> bool {}}-/// Continuously churn the worker pool until cancelled, affecting one worker at a-/// time so the pool always keeps a healthy majority (progress is always-/// possible). Drain = leave/rejoin the pool (EndpointSlice-style); Fail = stay-/// ready but error sends (a crash mid-send). Restores all workers on exit.+/// Churn one worker at a time until cancelled, then restore all.fn spawn_churn(harness: &Harness,churn: Churn,@@ -3907,9 +3877,7 @@ async fn run_churn_suite(churn: Churn, max_in_flight: usize, strict_order: bool)}}- // Once everything is delivered and churn has quiesced, the batcher's- // bookkeeping must drain back to zero. A leak here permanently skews- // placement (phantom load) or strands messages (pending forever).+ // Leaked bookkeeping would skew placement or strand messages.wait_until(Duration::from_secs(10),"batcher pending/in-flight to drain to zero after churn",@@ -3940,10 +3908,6 @@ async fn churn_mixed_drain_and_failures_are_exactly_once_at_max_in_flight_3() {run_churn_suite(Churn::Mixed, 3, false).await;}-/// A flapping worker — ready (in the pool) but failing every send — must not pin-/// a batch's offsets forever. The flush loop enforces its deadline on the-/// re-defer path too, so the consumer fails the batch within the configured-/// flush timeout instead of spinning. Runs at max_in_flight > 1.#[tokio::test]async fn flapping_worker_does_not_pin_batch_past_flush_timeout() {let topic = format!("e2e-flap-{}", Uuid::new_v4());@@ -3974,9 +3938,7 @@ async fn flapping_worker_does_not_pin_batch_past_flush_timeout() {let target = sole_arrived_worker(&harness);let other = 1 - target;- // `target` flaps: stays ready (in the pool) but fails every send. `other` is- // taken out of the pool so the flush can't escape — the flush loop keeps- // re-routing to the flapping target and re-deferring on the scatter path.+ // target stays ready but nacks; other leaves the pool, so the flush has nowhere else to go.harness.workers[target].ingest_ok.store(false, Ordering::SeqCst);@@ -3986,8 +3948,6 @@ async fn flapping_worker_does_not_pin_batch_past_flush_timeout() {guards[other] = None;guards[target] = None;- // The consumer must fail the batch within ~the flush timeout, not spin- // forever on the re-defer path.assert!(harness.wait_for_consumer_exit(Duration::from_secs(20))@@ -4001,11 +3961,6 @@ async fn flapping_worker_does_not_pin_batch_past_flush_timeout() {);}-// ── Worker stream fencing ──────────────────────────────────────────────────--/// A worker that nacks fences its worker stream: the fenced messages fail back into-/// the deferral path with nothing lost, and once the worker leaves the pool-/// the keys re-route to the survivor in order.#[tokio::test]async fn nacking_worker_fences_worker_stream_and_reroutes_in_order() {let topic = format!("e2e-fence-{}", Uuid::new_v4());@@ -4027,8 +3982,7 @@ async fn nacking_worker_fences_worker_stream_and_reroutes_in_order() {}harness.wait_for(8, Duration::from_secs(10)).await;- // Take down the worker that owns user-1: its stream nacks (fencing the- // worker stream) and its /_ready fails (dropping it from the pool and its pins).+ // Take down user-1's worker: it nacks and leaves the pool.let dead_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {0} else {@@ -4056,14 +4010,6 @@ async fn nacking_worker_fences_worker_stream_and_reroutes_in_order() {harness.stop().await;}-/// One routing key arriving on two partitions: a partition-count change-/// leaves a key's backlog on its old partition while new messages land on the-/// new one. The batcher merges such a key into one run (its key table is-/// partition-blind), so it must split the completion back per partition —-/// mis-attributed coverage would leave the poll uncommitted-/// and wedge the consumer. Two waves prove commits keep flowing: with-/// max_in_flight = 1, the second wave can only deliver after the first wave's-/// polls completed.#[tokio::test]async fn same_key_across_partitions_completes_and_commits() {let topic = format!("e2e-cross-partition-key-{}", Uuid::new_v4());@@ -4094,8 +4040,6 @@ async fn same_key_across_partitions_completes_and_commits() {}harness.wait_for(12, Duration::from_secs(15)).await;- // Everything delivered exactly once; cross-partition interleaving is- // free, but each partition's own sequence must arrive in order.let seqs = harness.workers[0].seqs_for("user-1");let mut sorted = seqs.clone();sorted.sort_unstable();diff --git a/rust/ingestion-consumer/tests/grpc_transport_test.rs b/rust/ingestion-consumer/tests/grpc_transport_test.rsindex 49a7f725179c..bf8e2001c5d4 100644--- a/rust/ingestion-consumer/tests/grpc_transport_test.rs+++ b/rust/ingestion-consumer/tests/grpc_transport_test.rs@@ -1,10 +1,3 @@-//! Worker stream-level tests for the gRPC transport against an in-process mock worker.-//!-//! Each test targets one property the worker stream must hold for per-key ordering:-//! wire order equals enqueue order, acks correlate out of order, and a-//! failure fences everything outstanding (in order, with the messages handed-//! back) so the batcher can requeue and replay it.-mod common;use std::collections::HashMap;@@ -34,23 +27,15 @@ use tokio_stream::wrappers::UnboundedReceiverStream;use tokio_stream::StreamExt;use tonic::{Request, Response, Status, Streaming};-/// What the mock does with each sub-batch it reads.#[derive(Clone, Copy)]enum AckMode {- /// Ack immediately, in read order.Immediate,- /// Buffer everything; the test triggers acks explicitly.Manual,- /// Nack the sub-batch with this seq (and ack the others immediately).NackSeq(u64),- /// Report busy for the sub-batch with this seq (and ack the others).BusySeq(u64),- /// Ack every sub-batch immediately except this seq, which is never acked —- /// a stuck oldest entry while its siblings keep acking.AckExceptSeq(u64),}-/// Manual mode: the test drives the worker's acks through this.enum ManualAck {Ok { seq: u64, accepted: u32 },Nack(u64),@@ -61,9 +46,7 @@ type ManualAcks = Arc<Mutex<Option<mpsc::UnboundedReceiver<ManualAck>>>>;struct MockWorker {mode: AckMode,- /// Sub-batches in the order the worker read them, across all streams.received: Arc<Mutex<Vec<SubBatch>>>,- /// Hellos in connection order (stream epochs prove reconnects).hellos: Arc<Mutex<Vec<u64>>>,manual_acks: ManualAcks,}@@ -84,8 +67,7 @@ impl WorkerIngest for MockWorker {let manual = self.manual_acks.lock().await.take();tokio::spawn(async move {- // Mirror the real worker: greet with `ready` so response headers- // flush; the worker stream must ignore it.+ // The real worker greets with `ready` to flush headers; the stream ignores it.let _ = tx.send(Ok(IngestStreamResponse {msg: Some(ingest_stream_response::Msg::Ready(StreamReady {})),}));@@ -255,10 +237,10 @@ fn msg(distinct_id: &str, offset: i64) -> SerializedKafkaMessage {key: Some(distinct_id.to_string()),value: Some("{}".to_string()),headers,+ generation: 0,}}-/// The worker stream's worker URL: HTTP port is fake, the gRPC port is the mock's.fn worker_url(addr: SocketAddr) -> String {format!("http://{}:9001", addr.ip())}@@ -269,8 +251,7 @@ async fn sub_batches_reach_the_worker_in_enqueue_order() {let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);- // Enqueue three sub-batches back to back, so ordering must survive the- // queue draining onto the stream in one go.+ // Three back-to-back sends drain onto the stream in one go, in order.let pending: Vec<_> = (0..3).map(|i| transport.begin_send(&url, &format!("batch-{i}"), vec![msg("d1", i)], false)).collect();@@ -299,9 +280,7 @@ async fn out_of_order_acks_resolve_the_right_sends() {let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1), msg("d1", 2)], false);let second = transport.begin_send(&url, "batch-2", vec![msg("d2", 3)], false);- // Wait until both are on the wire, then ack seq 2 before seq 1 with- // distinct counts — each pending must get its own, and the later one- // must not resolve ahead of the earlier one.+ // Ack seq 2 before seq 1.wait_for_received(&mock, 2).await;ack_tx.send(ManualAck::Ok {@@ -558,11 +537,7 @@ async fn a_budget_rides_only_a_single_frame_send(async fn a_failure_fences_a_later_sub_batch_the_worker_already_acked(#[case] second_ack: ManualAck,) {- // Regression: with more than one un-acked sub-batch, the worker may ack- // seq 2 before seq 1. If seq 2 resolved on its own ack, its caller would- // release its keys and never replay it, while seq 1's later failure- // requeued only seq 1 — the older messages would replay after the newer- // ones. A failure must fence the acked-but-unresolved tail too.+ // Else seq 2's keys would release and replay ahead of seq 1's.let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));@@ -603,11 +578,7 @@ async fn a_failure_fences_a_later_sub_batch_the_worker_already_acked(#[tokio::test]async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {- // Regression: a fence resolves its sends before their callers requeue the- // messages. In that gap the consumer loop can still enqueue a fenced- // key's next group; if the next stream sent it, it would reach the worker- // ahead of the requeued older messages. The worker stream must keep fencing until- // every fenced caller drops its guard.+ // Else the next send could reach the worker before the requeued messages.let mock = start_mock(AckMode::NackSeq(1), None).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);@@ -618,8 +589,6 @@ async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {.fence_guard.expect("a fenced send carries a guard");- // Enqueued while the fence is unacknowledged: fails without riding a- // stream.let second = transport.begin_send(&url, "batch-2", vec![msg("d1", 2)], false);let second_err = tokio::time::timeout(Duration::from_secs(2), second.wait()).await@@ -632,8 +601,6 @@ async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {));assert_eq!(mock.received.lock().await.len(), 1);- // Both callers requeued: the worker stream resumes and the next send reaches the- // worker on a fresh stream.drop(guard);drop(second_err);let third = transport.begin_send(&url, "batch-3", vec![msg("d1", 3)], false);@@ -652,14 +619,11 @@ async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {#[tokio::test]async fn an_oversized_sub_batch_rides_the_stream_as_ordered_chunks() {- // The worker caps one frame; a sub-batch over the cap must split into- // consecutive frames on the same stream instead of one frame the worker- // rejects, fences, and replays forever.+ // One frame over the worker's cap would fence and replay forever.let mock = start_mock(AckMode::Immediate, None).await;let mut transport =GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));- // Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this- // fits exactly one message per frame and three messages become three frames.+ // One `msg()` estimates to ~143 bytes, so each 200-byte frame fits one message.transport.set_max_body_bytes(200);let url = worker_url(mock.addr);@@ -699,14 +663,11 @@ async fn an_oversized_sub_batch_rides_the_stream_as_ordered_chunks() {#[tokio::test]async fn a_failed_chunk_hands_back_the_whole_sub_batch() {- // All-or-nothing like the HTTP path: chunk 1 is acked, chunk 2 nacked, so- // the send fails with every message (acked ones included) for deferral,- // and carries a fence guard.+ // One nacked chunk fails the whole send, acked chunks included.let mock = start_mock(AckMode::NackSeq(2), None).await;let mut transport =GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));- // Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this- // fits exactly one message per frame and three messages become three frames.+ // One `msg()` estimates to ~143 bytes, so each 200-byte frame fits one message.transport.set_max_body_bytes(200);let url = worker_url(mock.addr);@@ -761,16 +722,12 @@ async fn chunks_acked_out_of_order_then_fenced_hand_back_messages_in_send_order(#[tokio::test]async fn a_nack_fences_everything_outstanding_in_order() {- // Regression: on a failure, every un-acked and queued sub-batch must fail- // back to the caller with its messages (for the deferral path), and- // nothing may be silently retried on the next stream — a later sub-batch- // surviving the fence would leapfrog the failed one and reorder its keys.+ // A surviving later sub-batch would overtake the failed one.let mock = start_mock(AckMode::NackSeq(1), None).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);- // All three ride the stream at once, and the nack of the first must fence- // the other two in order.+ // All three ride the stream at once; the first's nack fences the other two in order.let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);let second = transport.begin_send(&url, "batch-2", vec![msg("d1", 2)], false);let third = transport.begin_send(&url, "batch-3", vec![msg("d2", 3)], false);@@ -807,12 +764,7 @@ async fn the_worker_stream_reconnects_with_a_new_stream_epoch_after_a_fence() {let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);first.wait().await.expect_err("nacked");- // The retry (as the deferral path would issue it) lands on a fresh stream- // with a bumped epoch and replay=true, and succeeds: seq 1 is nacked only- // once because NackSeq(1) nacks by seq, proving this send is seq 1 on a- // NEW stream... so use a different worker mode expectation: the second- // stream's seq-1 sub-batch is nacked again by this mock, which is fine —- // what we assert is the reconnect (two hellos, increasing epochs).+ // The mock nacks seq 1 on every stream; only the reconnect is asserted.let retry = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], true);retry.wait()@@ -829,9 +781,7 @@ async fn the_worker_stream_reconnects_with_a_new_stream_epoch_after_a_fence() {#[tokio::test]async fn a_dead_worker_fences_instead_of_hanging() {- // Connect failure (nothing listening): the send must resolve with its- // messages rather than waiting forever — the caller's deferral path owns- // the retry pacing.+ // The caller's deferral path owns the retry pacing.let transport = GrpcTransport::new(GrpcPort::Fixed(1), Duration::from_secs(30));let pending = transport.begin_send("http://127.0.0.1:9001",@@ -848,10 +798,6 @@ async fn a_dead_worker_fences_instead_of_hanging() {#[tokio::test]async fn a_worker_that_stops_acking_fences_after_the_watchdog_window() {- // Regression: a worker at capacity (or wedged) simply never acks, and the- // worker stream has no per-send timeout — without the watchdog the consumer waits- // forever and the whole pod wedges, as seen in production. The fence must- // hand the messages back so the deferral path re-routes them.let (_ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(@@ -871,12 +817,7 @@ async fn a_worker_that_stops_acking_fences_after_the_watchdog_window() {#[tokio::test]async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {- // Regression: the watchdog must bound each sub-batch's own wait, not the- // worker stream's time since its last ack. With more than one un-acked sub-batch, a- // worker that keeps acking newer sub-batches but never the oldest used to- // reset a single shared deadline on every ack, so the stuck send waited- // forever and its Kafka batch never completed — the wedge the watchdog- // exists to prevent.+ // Sibling acks must not extend the oldest sub-batch's deadline.let mock = start_mock(AckMode::AckExceptSeq(1), None).await;let transport = Arc::new(GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()),@@ -887,9 +828,7 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {// Enqueued first, so it is seq 1 — the one the worker never acks.let stuck = transport.begin_send(&url, "batch-stuck", vec![msg("d1", 1)], false);- // Keep feeding siblings faster than the ack timeout. Each is acked and- // drained (the worker stream holds at most one alongside the stuck entry), so a- // watchdog keyed on the last ack would never fire.+ // Feed acked siblings faster than the ack timeout.let feeder = {let transport = Arc::clone(&transport);let url = url.clone();@@ -906,8 +845,7 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {})};- // The stuck send must fence on its own deadline (~500ms), well before the- // feed stops (~3.2s) — proving sibling acks do not extend its wait.+ // Fences at ~500ms, before the feed stops at ~3.2s (40 x 80ms).let err = tokio::time::timeout(Duration::from_secs(2), stuck.wait()).await.expect("stuck send must fence on its own deadline, not wait for the feed to stop")@@ -920,9 +858,7 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {#[tokio::test]async fn removing_a_worker_lets_its_in_flight_send_settle_by_ack() {- // Regression: a reaped worker that is slow rather than dead still finishes- // its request. Fencing the send on removal made a survivor repeat it while- // the reaped worker still ran it.+ // A reaped worker that is slow, not dead, still finishes its request; a fence would make a survivor repeat it.let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));@@ -947,10 +883,7 @@ async fn removing_a_worker_lets_its_in_flight_send_settle_by_ack() {#[tokio::test]async fn removing_a_worker_resolves_its_unacked_send_by_the_watchdog() {- // Regression: reaping a worker that still has in-flight work must resolve- // its un-acked sends with the messages intact, so the deferral path can- // replay them. Aborting the worker stream task instead dropped the sends unresolved,- // which the caller could only recover by crashing and replaying.+ // Else the caller recovers the dropped send only by crashing and replaying.let (_ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(@@ -960,7 +893,6 @@ async fn removing_a_worker_resolves_its_unacked_send_by_the_watchdog() {let url = worker_url(mock.addr);let pending = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);- // Wait until the send is on the wire (in the worker stream's ledger) before reaping.for _ in 0..200 {if mock.received.lock().await.len() == 1 {break;@@ -985,9 +917,7 @@ async fn removing_a_worker_resolves_its_unacked_send_by_the_watchdog() {#[tokio::test]async fn a_busy_status_fences_as_retriable_with_messages() {- // A worker reporting busy is applying backpressure, not failing: the fenced- // sends must carry their messages (to replay) and classify as retriable, so- // the consumer re-routes without counting the worker as unhealthy.+ // Busy is backpressure: re-route without marking the worker unhealthy.let mock = start_mock(AckMode::BusySeq(1), None).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);@@ -1130,7 +1060,7 @@ async fn the_watchdog_bounds_overlapping_busy_retries() {let mut accumulator = Accumulator::default();accumulator.push(Partition(0), msg("a", 1).into());accumulator.push(Partition(0), msg("b", 2).into());- batcher.submit(accumulator);+ batcher.submit(accumulator, &HashMap::new());let first = tokio::time::timeout(Duration::from_secs(1), attempts_rx.recv()).await@@ -1145,10 +1075,7 @@ async fn the_watchdog_bounds_overlapping_busy_retries() {"the two keys must occupy different workers");- // Always leave one request outstanding while rejecting the other. If a- // retry does not arrive, release the held request after a bounded delay:- // the watchdog may wait for a slow in-flight attempt to settle, but must- // then diagnose the scheduler-wide lack of acceptance.+ // Keep one request held; release it if no retry arrives in time.let busy_replies = Arc::new(AtomicUsize::new(0));let replies = Arc::clone(&busy_replies);let final_attempt_released = Arc::new(AtomicBool::new(false));@@ -1212,7 +1139,7 @@ async fn a_busy_send_is_retried_until_it_completes() {let mut accumulator = Accumulator::default();accumulator.push(Partition(0), msg("a", 1).into());- batcher.submit(accumulator);+ batcher.submit(accumulator, &HashMap::new());let first = tokio::time::timeout(Duration::from_secs(1), attempts_rx.recv()).await@@ -1257,7 +1184,7 @@ async fn the_watchdog_allows_in_flight_success_after_the_deadline() {let mut accumulator = Accumulator::default();accumulator.push(Partition(0), msg("a", 1).into());- batcher.submit(accumulator);+ batcher.submit(accumulator, &HashMap::new());let attempt = tokio::time::timeout(Duration::from_secs(1), attempts_rx.recv()).await