:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): commit revoked partitions before releasing them (#126503)
Paweł Ledwoń250c0af11fc2
diff --git a/rust/ingestion-consumer/src/batcher/drains.rs b/rust/ingestion-consumer/src/batcher/drains.rsnew file mode 100644index 000000000000..8c03d90a8572--- /dev/null+++ b/rust/ingestion-consumer/src/batcher/drains.rs@@ -0,0 +1,37 @@+use std::collections::HashSet;++use super::in_flight::RequestId;+use super::RevokeId;++#[derive(Default)]+pub struct PartitionDrains {+ pending: Vec<PartitionDrain>,+}++struct PartitionDrain {+ revoke: RevokeId,+ carrying: HashSet<RequestId>,+}++impl PartitionDrains {+ pub fn on_revoked(&mut self, revoke: RevokeId, carrying: impl IntoIterator<Item = RequestId>) {+ self.pending.push(PartitionDrain {+ revoke,+ carrying: carrying.into_iter().collect(),+ });+ }++ pub fn on_request_settled(&mut self, request: RequestId) {+ for drain in &mut self.pending {+ drain.carrying.remove(&request);+ }+ }++ pub fn take_drained(&mut self) -> Vec<RevokeId> {+ let (drained, pending): (Vec<_>, Vec<_>) = std::mem::take(&mut self.pending)+ .into_iter()+ .partition(|drain| drain.carrying.is_empty());+ self.pending = pending;+ drained.into_iter().map(|drain| drain.revoke).collect()+ }+}diff --git a/rust/ingestion-consumer/src/batcher/driver.rs b/rust/ingestion-consumer/src/batcher/driver.rsindex 88b775bab316..f059a0903686 100644--- a/rust/ingestion-consumer/src/batcher/driver.rs+++ b/rust/ingestion-consumer/src/batcher/driver.rs@@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};use std::time::{Duration, Instant};-use common_kafka_consumer::{AssignmentEpoch, GroupCompletion, Partition};+use common_kafka_consumer::{AssignmentEpoch, Partition};use futures::future::{BoxFuture, FutureExt};use futures::stream::{FuturesUnordered, StreamExt};use metrics::{counter, histogram};@@ -20,7 +20,7 @@ use super::state_machine::{BatcherSnapshot, BatcherStateMachine, Effects, FailureCause, SendRequest,};use super::worker_pool::WorkerPoolSource;-use super::{make_batch_id, BatcherOutputs};+use super::{make_batch_id, BatcherCompletion, BatcherOutputs, RevokeId};use crate::debug_recorder::{partition_offsets, record_if, DebugEventKind, DebugRecorder};use crate::grpc_transport::{GrpcTransport, SendOptions};use crate::order_sentinel::{KeyOrderSentinel, SendKind, SentinelBatch};@@ -44,7 +44,10 @@ enum Input {assignment_epoch: u64,runs: Vec<KeyRun>,},- PartitionsRevoked(Vec<(String, i32)>),+ PartitionsRevoked {+ revoke: RevokeId,+ partitions: Vec<(String, i32)>,+ },Shutdown,/// Answered between two actions, so the snapshot is consistent.Snapshot(oneshot::Sender<BatcherSnapshot>),@@ -60,8 +63,6 @@ pub(super) struct Load {pub adaptation: Adaptation,}-pub(super) struct RevokeSender(mpsc::UnboundedSender<Input>);-#[derive(Clone)]pub(super) struct SnapshotSender(mpsc::UnboundedSender<Input>);@@ -140,7 +141,7 @@ struct BatcherTask<S, W> {workers: W,sender: Arc<S>,key_sentinel: Arc<KeyOrderSentinel>,- completions: mpsc::UnboundedSender<GroupCompletion>,+ completions: mpsc::UnboundedSender<BatcherCompletion>,errors: mpsc::UnboundedSender<String>,load: Arc<Load>,recorder: Option<Arc<DebugRecorder>>,@@ -207,10 +208,6 @@ impl StateMachineDriver {Arc::clone(&self.load)}- pub(super) fn revoke_sender(&self) -> RevokeSender {- RevokeSender(self.inputs.clone())- }-pub(super) fn snapshot_sender(&self) -> SnapshotSender {SnapshotSender(self.inputs.clone())}@@ -243,6 +240,13 @@ impl StateMachineDriver {self.send(Input::Shutdown);}+ pub(super) fn revoke(&self, revoke: RevokeId, partitions: &[(String, i32)]) {+ self.send(Input::PartitionsRevoked {+ revoke,+ partitions: partitions.to_vec(),+ });+ }+fn send(&self, input: Input) {if self.inputs.send(input).is_err() {error!("Batcher task is gone");@@ -256,15 +260,6 @@ impl Drop for StateMachineDriver {}}-impl RevokeSender {- /// Called from the rebalance callback inside the consumer loop's Kafka- /// poll, so it queues the revoke rather than waiting for it. The revoke- /// lands between the polls submitted before and after the rebalance.- pub(super) fn purge_revoked(&self, partitions: &[(String, i32)]) {- let _ = self.0.send(Input::PartitionsRevoked(partitions.to_vec()));- }-}-impl<S: RequestSender, W: Workers> BatcherTask<S, W> {async fn run(mut self, mut state: BatcherStateMachine) {loop {@@ -306,11 +301,11 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {assigned = true;state.on_groups(now, &self.workers.candidates(), assignment_epoch, runs)}- Event::Input(Input::PartitionsRevoked(partitions)) => {+ Event::Input(Input::PartitionsRevoked { revoke, partitions }) => {// Cleared in order with the sends, so no revoked message is// noted as sent after the revoke.self.key_sentinel.clear();- state.on_partitions_revoked(now, &partitions)+ state.on_partitions_revoked(now, revoke, &partitions)}Event::Input(Input::Shutdown) => state.on_shutdown(now, &self.workers.candidates()),Event::Input(Input::Snapshot(_)) => unreachable!("answered above"),@@ -368,6 +363,7 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {let Effects {sends,completions,+ drained,key_acks,evicted_keys,worker_outcomes,@@ -408,7 +404,10 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {counter!("ingestion_consumer_group_completions_total").increment(1);counter!("ingestion_consumer_group_completion_accepted_messages_total").increment(completion.accepted as u64);- let _ = self.completions.send(completion);+ let _ = self.completions.send(BatcherCompletion::Group(completion));+ }+ for revoke in drained {+ let _ = self.completions.send(BatcherCompletion::Drained(revoke));}if let Some(reason) = fatal {error!(error = %reason, "Batcher failure");@@ -687,14 +686,38 @@ mod tests {}fn completed_offsets(&mut self) -> Vec<i64> {- let mut completed = Vec::new();+ self.received()+ .into_iter()+ .filter_map(|received| match received {+ Received::Completed(offset) => Some(offset),+ Received::Drained(_) => None,+ })+ .collect()+ }++ fn received(&mut self) -> Vec<Received> {+ let mut received = Vec::new();while let Ok(completion) = self.outputs.completions.try_recv() {- completed.extend(completion.offsets.iter().map(|offset| offset.0));+ match completion {+ BatcherCompletion::Group(group) => received.extend(+ group+ .offsets+ .iter()+ .map(|offset| Received::Completed(offset.0)),+ ),+ BatcherCompletion::Drained(revoke) => received.push(Received::Drained(revoke)),+ }}- completed+ received}}+ #[derive(Debug, PartialEq)]+ enum Received {+ Completed(i64),+ Drained(RevokeId),+ }+impl FakeWorkers {fn set_candidates(&self, workers: &[&str]) {self.0.lock().unwrap().candidates =@@ -869,9 +892,7 @@ mod tests {let send = h.take_sent().pop().unwrap();send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();- h.driver- .revoke_sender()- .purge_revoked(&[("events".to_string(), 0)]);+ h.driver.revoke(RevokeId(1), &[("events".to_string(), 0)]);settle().await;assert!(@@ -889,13 +910,54 @@ mod tests {let sentinel = h.driver.key_order_sentinel();assert_eq!(sentinel.key_count(), 1);- h.driver- .revoke_sender()- .purge_revoked(&[("events".to_string(), 9)]);+ h.driver.revoke(RevokeId(1), &[("events".to_string(), 9)]);settle().await;assert_eq!(sentinel.key_count(), 0);}+ #[tokio::test(start_paused = true)]+ async fn a_drain_signals_once_no_request_in_flight_carries_its_partitions() {+ let mut h = Harness::new(&["w"], 4);+ h.submit(vec![run("a", &[1])]);+ settle().await;+ let send = h.take_sent().pop().unwrap();+ let (idle, busy) = (RevokeId(1), RevokeId(2));++ h.driver.revoke(idle, &[("events".to_string(), 9)]);+ h.driver.revoke(busy, &[("events".to_string(), 0)]);+ settle().await;+ assert_eq!(h.received(), vec![Received::Drained(idle)]);++ send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();+ settle().await;+ assert_eq!(+ h.received(),+ vec![Received::Completed(1), Received::Drained(busy)]+ );+ }++ #[tokio::test(start_paused = true)]+ async fn a_drain_waits_for_a_poll_submitted_just_before_it() {+ let mut h = Harness::new(&["w"], 4);+ let revoke = RevokeId(1);++ h.submit(vec![run("a", &[1])]);+ h.driver.revoke(revoke, &[("events".to_string(), 0)]);+ settle().await;+ let send = h+ .take_sent()+ .pop()+ .expect("the poll went out before the drain");+ assert_eq!(h.received(), vec![]);++ send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();+ settle().await;+ assert_eq!(+ h.received(),+ vec![Received::Completed(1), Received::Drained(revoke)]+ );+ }+#[tokio::test(start_paused = true)]async fn the_load_tracks_pending_in_flight_and_busy_workers() {let h = Harness::new(&[], 2);diff --git a/rust/ingestion-consumer/src/batcher/in_flight.rs b/rust/ingestion-consumer/src/batcher/in_flight.rsindex f8836955ce37..3e91e8d7f91e 100644--- a/rust/ingestion-consumer/src/batcher/in_flight.rs+++ b/rust/ingestion-consumer/src/batcher/in_flight.rs@@ -127,6 +127,25 @@ impl InFlightRequests {pub fn take(&mut self, id: RequestId) -> Option<InFlightRequest> {self.requests.remove(&id)}++ /// Matches partition numbers only, because a sent message carries no topic.+ pub fn carrying<'a>(+ &'a self,+ partitions: &'a [(String, i32)],+ ) -> impl Iterator<Item = RequestId> + 'a {+ self.requests+ .iter()+ .filter(|(_, request)| {+ request.runs.iter().any(|run| {+ run.messages.iter().any(|message| {+ partitions+ .iter()+ .any(|(_, partition)| *partition == message.partition)+ })+ })+ })+ .map(|(id, _)| *id)+ }}impl InFlightRequest {diff --git a/rust/ingestion-consumer/src/batcher/mod.rs b/rust/ingestion-consumer/src/batcher/mod.rsindex ab9660a644de..e7657de9f5bb 100644--- a/rust/ingestion-consumer/src/batcher/mod.rs+++ b/rust/ingestion-consumer/src/batcher/mod.rs@@ -1,11 +1,12 @@//! The worker batcher: the dispatch orchestration behind one boundary.//!//! The consumer loop submits one [`Accumulator`] per poll, in poll order, and-//! receives [`GroupCompletion`]s back, one per partition and generation of each-//! settled request. Everything in between is an implementation detail of this-//! module: the state machine queues each key's messages, packs ready keys into-//! requests for free worker slots, places each request on a worker as it is-//! sent, and retries failures; one task drives it and performs its effects. No+//! receives [`BatcherCompletion`]s back: one per partition and generation of+//! each settled request, and one per revoke. Everything in between is an+//! implementation detail of this module: the state machine queues each key's+//! messages, packs ready keys into requests for free worker slots, places each+//! request on a worker as it is sent, and retries failures; one task drives it+//! and performs its effects. No//! batch identity crosses the boundary: the consumer correlates completions by//! partition and offset.//!@@ -14,6 +15,7 @@//! are reported on an error channel; the consumer turns them into a process//! failure, so the failure decision stays in the consumer loop.+mod drains;mod driver;pub mod governor;pub mod in_flight;@@ -44,10 +46,18 @@ use crate::order_sentinel::KeyOrderSentinel;use crate::types::Accumulator;use crate::worker_registry::WorkerId;+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub struct RevokeId(pub u64);++pub enum BatcherCompletion {+ /// One partition and generation of a settled request.+ Group(GroupCompletion),+ Drained(RevokeId),+}+/// The batcher's output channels, held by the consumer loop.pub struct BatcherOutputs {- /// One completion per partition and generation of each settled request.- pub completions: mpsc::UnboundedReceiver<GroupCompletion>,+ pub completions: mpsc::UnboundedReceiver<BatcherCompletion>,/// Fatal orchestration failures.pub errors: mpsc::UnboundedReceiver<String>,}@@ -73,9 +83,12 @@ impl Batcher {self.driver.key_order_sentinel()}- /// The batcher's half of the consumer's revocation hook.- pub fn revoker(&self) -> Revoker {- Revoker(self.driver.revoke_sender())+ /// Drops the partitions' queued messages, also those of polls submitted+ /// before this call, and the messages that come back unfinished or failed.+ /// Answers with [`BatcherCompletion::Drained`] after the last `Group` of+ /// the partitions.+ pub fn revoke(&self, revoke: RevokeId, partitions: &[(String, i32)]) {+ self.driver.revoke(revoke, partitions);}/// Submit one poll's demuxed groups. Call on the consumer loop, in poll@@ -142,16 +155,6 @@ impl BatcherObserver {}}-/// Purges revoked partitions from the batcher. Purged messages never-/// complete.-pub struct Revoker(driver::RevokeSender);--impl Revoker {- pub fn purge_revoked(&self, partitions: &[(String, i32)]) {- self.0.purge_revoked(partitions);- }-}-pub(crate) fn make_batch_id() -> String {let ts = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)diff --git a/rust/ingestion-consumer/src/batcher/state_machine.rs b/rust/ingestion-consumer/src/batcher/state_machine.rsindex 6acf2fc5e58f..96eb1d068774 100644--- a/rust/ingestion-consumer/src/batcher/state_machine.rs+++ b/rust/ingestion-consumer/src/batcher/state_machine.rs@@ -5,9 +5,9 @@//!//! The caller performs one action's effects in this order: order-sentinel//! ACKs before evictions, then the sends in the order given, which is each-//! key's send order. `next_wakeup` replaces the previous wakeup; `None` needs-//! no timer. `fatal` means the state machine failed and the process must-//! exit and replay.+//! key's send order, and `drained` after `completions`. `next_wakeup`+//! replaces the previous wakeup; `None` needs no timer. `fatal` means the+//! state machine failed and the process must exit and replay.//!//! When work is pending, nothing is in flight, and no message was accepted//! for `stall_timeout`, the state machine fails, so a wedged batcher restarts@@ -21,6 +21,7 @@ use std::time::{Duration, Instant};use common_kafka_consumer::{GroupCompletion, Offset, Partition};use metrics::{counter, gauge, histogram};+use super::drains::PartitionDrains;use super::governor::{Adaptation, Governor, GovernorSettings, GovernorSnapshot};use super::in_flight::{InFlightRequest, InFlightRequests, RequestId, SentMessage};use super::key_queues::{KeyQueues, KeyRun, Settlement};@@ -28,6 +29,7 @@ use super::packer::Packer;use super::request::{Request, RequestClass};use super::retry_policy::{RetryPolicy, RetryReason};use super::worker_assigner::{WorkerAssigner, WorkerInFlight};+use super::RevokeId;use crate::order_sentinel::ack_mark;use crate::types::SerializedKafkaMessage;use crate::worker_registry::WorkerId;@@ -73,6 +75,7 @@ pub struct WorkerOutcome {pub struct Effects {pub sends: Vec<SendRequest>,pub completions: Vec<GroupCompletion>,+ pub drained: Vec<RevokeId>,pub key_acks: Vec<KeyAck>,pub evicted_keys: Vec<Arc<str>>,pub worker_outcomes: Vec<WorkerOutcome>,@@ -143,6 +146,7 @@ impl BatcherStateMachine {soft_budget: None,governor,reconnecting: HashMap::new(),+ drains: PartitionDrains::default(),}))}@@ -245,13 +249,16 @@ impl BatcherStateMachine {/// The new owner replays the revoked partitions, so their pending/// messages drop. Runs already in flight finish; their messages of revoked- /// partitions that come back unfinished or failed drop too.+ /// partitions that come back unfinished or failed drop too. `revoke`+ /// comes back in [`Effects::drained`] once no request in flight carries+ /// the partitions.pub fn on_partitions_revoked(self,now: Instant,+ revoke: RevokeId,partitions: &[(String, i32)],) -> (Self, Effects) {- self.act(|active, _| active.on_partitions_revoked(now, partitions))+ self.act(|active, _| active.on_partitions_revoked(now, revoke, partitions))}/// Takes no new groups from now on, and stops once nothing is pending or@@ -350,6 +357,7 @@ pub struct ActiveState {/// may wait it out and is no RTT sample. The first such request to settle/// clears the mark.reconnecting: HashMap<WorkerId, Instant>,+ drains: PartitionDrains,}impl ActiveState {@@ -532,12 +540,16 @@ impl ActiveState {fn on_partitions_revoked(&mut self,now: Instant,+ revoke: RevokeId,partitions: &[(String, i32)],) -> Result<Effects, String> {let mut effects = Effects {evicted_keys: self.keys.purge(partitions),..Effects::default()};+ // After the purge no new request carries the partitions.+ self.drains+ .on_revoked(revoke, self.in_flight.carrying(partitions));self.finish(now, &mut effects)?;if self.keys.has_ready() {effects.next_wakeup = Some(now);@@ -554,6 +566,7 @@ impl ActiveState {.in_flight.take(request).ok_or_else(|| format!("response for unknown request {request:?}"))?;+ self.drains.on_request_settled(request);if self.assigner.release(&sent.worker, sent.message_count) {effects.idle_workers.push(sent.worker.clone());}@@ -706,6 +719,7 @@ impl ActiveState {}fn finish(&mut self, now: Instant, effects: &mut Effects) -> Result<(), String> {+ effects.drained.extend(self.drains.take_drained());// A stalled action and a revoke skip `pack`, so without this a due// retry would set the next wakeup at or before `now`.self.keys.promote_due(now);@@ -1185,7 +1199,8 @@ mod tests {);let late = now + BUSY_DELAY * 3;- let (_, effects) = batcher.on_partitions_revoked(late, &[("events".to_string(), 9)]);+ let (_, effects) =+ batcher.on_partitions_revoked(late, REVOKE, &[("events".to_string(), 9)]);assert_eq!(effects.next_wakeup, Some(late));}@@ -1246,7 +1261,8 @@ mod tests {let (batcher, effects) = batcher.on_groups(now, &pool(&[]), 0, vec![run("a", &[1])]);assert!(effects.sends.is_empty());- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, effects) =+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);assert!(effects.sends.is_empty());assert_eq!(effects.evicted_keys, vec![Arc::<str>::from("a")]);assert_eq!(batcher.pending_messages(), 0);@@ -1710,7 +1726,8 @@ mod tests {let (batcher, effects) = batcher.on_groups(now, &workers, 0, vec![run("a", &[1])]);assert!(effects.sends.is_empty());- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, effects) =+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);assert_eq!(effects.evicted_keys, vec![Arc::<str>::from("a")]);assert_eq!(batcher.pending_messages(), 0);@@ -1882,7 +1899,8 @@ mod tests {let (batcher, effects) = batcher.on_groups(now, &workers, 0, vec![later]);assert!(effects.sends.is_empty());- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, effects) =+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);assert!(effects.evicted_keys.is_empty(), "a's run is in flight");let (batcher, _) =batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);@@ -1910,7 +1928,7 @@ mod tests {vec![run("a", &[1, 2])],);let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));- let (batcher, _) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);// The new owner of the partition replays the uncommitted offsets.let mut replay = run("a", &[1, 2, 3]);for message in &mut replay.messages {@@ -2209,7 +2227,8 @@ mod tests {let (batcher, _) =batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, effects) =+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);assert_eq!(effects.next_wakeup, Some(now), "a is ready again at once");let (_, effects) = batcher.on_wakeup(now, &workers);assert!(!effects.sends[0].class.replay);@@ -2258,7 +2277,7 @@ mod tests {let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));let (batcher, _) = batcher.on_shutdown(now, &workers);- let (batcher, _) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);assert!(matches!(batcher, BatcherStateMachine::Draining(_)),"a is in flight"@@ -2271,4 +2290,211 @@ mod tests {assert!(effects.completions.is_empty());assert_eq!(effects.next_wakeup, None);}++ #[test]+ fn a_partial_response_of_a_draining_partition_completes_only_what_was_accepted() {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = packing_batcher(3, Duration::ZERO, 4, now).on_groups(+ now,+ &workers,+ 0,+ vec![run("a", &[1, 2, 3])],+ );+ let request = effects.sends[0].request;+ let (batcher, effects) =+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);+ assert!(effects.drained.is_empty(), "a is in flight");++ let (_, effects) =+ batcher.on_request_partial(now, &workers, request, 2, vec![message("a", 0, 2)]);+ assert_eq!(effects.completions[0].offsets, vec![Offset(1), Offset(3)]);+ assert!(+ effects.sends.is_empty(),+ "the unfinished offset is not resent"+ );+ assert_eq!(effects.drained, vec![REVOKE]);+ }++ const REVOKE: RevokeId = RevokeId(1);+ const OTHER_REVOKE: RevokeId = RevokeId(2);++ fn partition(number: i32) -> Vec<(String, i32)> {+ vec![("events".to_string(), number)]+ }++ fn key_run(key: &str, messages: &[(i32, i64)]) -> KeyRun {+ KeyRun {+ routing_key: key.into(),+ messages: messages+ .iter()+ .map(|&(partition, offset)| message(key, partition, offset))+ .collect(),+ }+ }++ fn request_of(effects: &Effects, key: &str) -> RequestId {+ effects+ .sends+ .iter()+ .find(|send| &*send.runs[0].routing_key == key)+ .expect("a request for the key")+ .request+ }++ fn completed_offsets(effects: &Effects) -> Vec<i64> {+ effects+ .completions+ .iter()+ .flat_map(|completion| completion.offsets.iter().map(|offset| offset.0))+ .collect()+ }++ #[test]+ fn a_drain_waits_only_for_the_requests_carrying_its_partitions() {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = batcher(4, now).on_groups(+ now,+ &workers,+ 0,+ vec![+ key_run("a", &[(0, 1)]),+ key_run("b", &[(1, 7)]),+ key_run("c", &[(1, 8)]),+ ],+ );+ let (on_p0, on_p1) = (request_of(&effects, "a"), request_of(&effects, "b"));+ let (batcher, effects) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));+ assert!(effects.drained.is_empty());++ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, on_p1, 1);+ assert!(+ effects.drained.is_empty(),+ "partition 0's request is in flight"+ );++ let (_, effects) = batcher.on_request_succeeded(now, &workers, on_p0, 1);+ assert_eq!(+ effects.drained,+ vec![REVOKE],+ "c's request for partition 1 is still in flight"+ );+ }++ #[test]+ fn a_failed_run_spanning_a_draining_partition_ends_the_drain_and_replays_only_the_kept_partition(+ ) {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = packing_batcher(4, Duration::ZERO, 4, now).on_groups(+ now,+ &workers,+ 0,+ vec![key_run("a", &[(0, 1), (1, 7)])],+ );+ let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));++ let (batcher, effects) =+ batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);+ assert_eq!(+ effects.drained,+ vec![REVOKE],+ "partition 1's replay is not awaited"+ );++ let (_, effects) = batcher.on_wakeup(now + FAULT_DELAY, &workers);+ assert_eq!(shape(&effects.sends[0]), vec![("a", vec![7])]);+ }++ #[test]+ fn a_partial_response_spanning_a_draining_partition_resends_only_the_kept_partition() {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = packing_batcher(4, Duration::ZERO, 4, now).on_groups(+ now,+ &workers,+ 0,+ vec![key_run("a", &[(0, 1), (1, 7), (0, 2), (1, 8)])],+ );+ let request = effects.sends[0].request;+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));++ let (_, effects) = batcher.on_request_partial(+ now,+ &workers,+ request,+ 2,+ vec![message("a", 0, 2), message("a", 1, 8)],+ );+ assert_eq!(completed_offsets(&effects), vec![1, 7]);+ assert_eq!(shape(&effects.sends[0]), vec![("a", vec![8])]);+ assert_eq!(effects.drained, vec![REVOKE]);+ }++ #[test]+ fn two_drains_each_end_when_their_own_requests_settle() {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = batcher(4, now).on_groups(+ now,+ &workers,+ 0,+ vec![key_run("a", &[(0, 1)]), key_run("b", &[(1, 7)])],+ );+ let (on_p0, on_p1) = (request_of(&effects, "a"), request_of(&effects, "b"));+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));+ let (batcher, _) = batcher.on_partitions_revoked(now, OTHER_REVOKE, &partition(1));++ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, on_p1, 1);+ assert_eq!(effects.drained, vec![OTHER_REVOKE]);++ let (_, effects) = batcher.on_request_succeeded(now, &workers, on_p0, 1);+ assert_eq!(effects.drained, vec![REVOKE]);+ }++ fn queued_without_a_worker(now: Instant) -> BatcherStateMachine {+ let (batcher, _) = batcher(4, now).on_groups(now, &pool(&[]), 0, vec![run("a", &[1])]);+ batcher+ }++ fn waiting_for_a_retry(now: Instant) -> BatcherStateMachine {+ let workers = pool(&["w"]);+ let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);+ let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));+ let (batcher, _) =+ batcher.on_request_failed(now, &workers, request, FailureCause::Busy, messages);+ batcher+ }++ #[rstest]+ #[case::queued_without_a_worker(queued_without_a_worker)]+ #[case::waiting_for_a_retry(waiting_for_a_retry)]+ fn a_drain_with_nothing_in_flight_ends_at_once_and_its_messages_are_never_sent(+ #[case] setup: fn(Instant) -> BatcherStateMachine,+ ) {+ let now = Instant::now();+ let (batcher, effects) = setup(now).on_partitions_revoked(now, REVOKE, &partition(0));+ assert_eq!(effects.drained, vec![REVOKE]);++ let (_, effects) = batcher.on_wakeup(now + FAULT_DELAY, &pool(&["w"]));+ assert!(effects.sends.is_empty());+ }++ #[test]+ fn a_drain_during_shutdown_ends_with_the_last_request() {+ let now = Instant::now();+ let workers = pool(&["w"]);+ let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);+ let request = effects.sends[0].request;+ let (batcher, _) = batcher.on_shutdown(now, &workers);+ let (batcher, effects) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));+ assert!(effects.drained.is_empty());++ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, request, 1);+ assert!(matches!(batcher, BatcherStateMachine::Stopped));+ assert_eq!(completed_offsets(&effects), vec![1]);+ assert_eq!(effects.drained, vec![REVOKE]);+ }}diff --git a/rust/ingestion-consumer/src/config.rs b/rust/ingestion-consumer/src/config.rsindex 40051f022d43..d5b8bd17f38a 100644--- a/rust/ingestion-consumer/src/config.rs+++ b/rust/ingestion-consumer/src/config.rs@@ -18,6 +18,14 @@ use crate::partition_budget::BudgetLimits;use crate::routing::{Router, RoutingStrategy};use common_kafka_consumer::config::ConsumerConfigBuilder;+/// The consumer loop must report healthy within this, or the lifecycle+/// manager restarts the process.+pub const LIVENESS_DEADLINE: Duration = Duration::from_secs(60);++/// How long a shutdown waits for the consumer loop to drain its in-flight+/// work before the process exits.+pub const GRACEFUL_SHUTDOWN: Duration = Duration::from_secs(60);+/// Configuration for the ingestion consumer. Env var names and defaults match/// the Node.js ingestion consumer, so both read the same ConfigMap./// `KAFKA_CONSUMER_*` vars also apply as rdkafka overrides; see@@ -221,6 +229,17 @@ pub struct Config {#[envconfig(from = "CONSUMER_COMMIT_INTERVAL_MS", default = "500")]pub consumer_commit_interval_ms: u64,+ /// How long a revoke waits for the revoked partitions' in-flight messages+ /// before it commits them and hands the partitions to their new owner+ /// (milliseconds). The commit gets the same time again. librdkafka fetches+ /// no partition while it waits. `0` (default) hands them over at once,+ /// and the new owner processes their in-flight messages again. At least+ /// twice `INGESTION_REQUEST_SOFT_BUDGET_MS` when that is set, and at most+ /// half the smallest of the liveness deadline, the graceful shutdown, and+ /// `KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS`.+ #[envconfig(from = "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", default = "0")]+ pub consumer_revoke_drain_timeout_ms: u64,+/// Uncommitted events at which a partition's fetch pauses. Required.#[envconfig(from = "CONSUMER_PARTITION_BUDGET_EVENTS", default = "0")]pub consumer_partition_budget_events: u64,@@ -640,6 +659,48 @@ impl Config {}))}+ /// A budgeted request comes back within one soft budget. A drain shorter+ /// than two soft budgets can time out on requests that are about to finish.+ const MIN_REVOKE_DRAIN_SOFT_BUDGETS: u64 = 2;++ pub fn revoke_drain_timeout(&self) -> Result<Option<Duration>, String> {+ let timeout_ms = self.consumer_revoke_drain_timeout_ms;+ if timeout_ms == 0 {+ return Ok(None);+ }+ let floor_ms = self+ .request_soft_budget_ms+ .saturating_mul(Self::MIN_REVOKE_DRAIN_SOFT_BUDGETS);+ if timeout_ms < floor_ms {+ return Err(format!(+ "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS must be 0 or at least {} times \+ INGESTION_REQUEST_SOFT_BUDGET_MS ({floor_ms})",+ Self::MIN_REVOKE_DRAIN_SOFT_BUDGETS+ ));+ }+ // The drain's commit phase neither polls Kafka nor reports healthy for+ // up to the timeout, a drain at shutdown adds that phase to the+ // graceful budget, and wait plus commit must fit the broker's+ // rebalance timeout (max.poll.interval.ms).+ let ceiling_ms = [+ LIVENESS_DEADLINE.as_millis() as u64,+ GRACEFUL_SHUTDOWN.as_millis() as u64,+ u64::from(self.kafka_consumer_max_poll_interval_ms),+ ]+ .into_iter()+ .min()+ .expect("three candidates")+ / 2;+ if timeout_ms > ceiling_ms {+ return Err(format!(+ "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS must be at most half the smallest of the \+ liveness deadline, the graceful shutdown, and \+ KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS ({ceiling_ms})"+ ));+ }+ Ok(Some(Duration::from_millis(timeout_ms)))+ }+pub fn worker_urls(&self) -> Vec<String> {self.worker_addresses.split(',')@@ -779,6 +840,28 @@ mod tests {assert!(config(vars).request_budget().is_err());}+ struct DrainTimeoutCase {+ vars: &'static [(&'static str, &'static str)],+ expected: Result<Option<Duration>, ()>,+ }++ #[rstest]+ #[case::off(DrainTimeoutCase { vars: &[("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Ok(None) })]+ #[case::no_soft_budget(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "100")], expected: Ok(Some(Duration::from_millis(100))) })]+ #[case::at_two_soft_budgets(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "6000"), ("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Ok(Some(Duration::from_millis(6000))) })]+ #[case::below_two_soft_budgets(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "5999"), ("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Err(()) })]+ #[case::at_half_the_liveness_deadline(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "30000")], expected: Ok(Some(Duration::from_millis(30000))) })]+ #[case::above_half_the_liveness_deadline(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "30001")], expected: Err(()) })]+ #[case::above_half_a_short_poll_interval(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "10001"), ("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS", "20000")], expected: Err(()) })]+ fn the_revoke_drain_timeout_is_off_or_between_two_soft_budgets_and_half_the_tightest_budget(+ #[case] case: DrainTimeoutCase,+ ) {+ assert_eq!(+ config(case.vars).revoke_drain_timeout().map_err(drop),+ case.expected+ );+ }+struct LimitsCase {vars: &'static [(&'static str, &'static str)],expected_valid: bool,diff --git a/rust/ingestion-consumer/src/consumer.rs b/rust/ingestion-consumer/src/consumer.rsindex b1af291221db..c37cbf6c861a 100644--- a/rust/ingestion-consumer/src/consumer.rs+++ b/rust/ingestion-consumer/src/consumer.rs@@ -1,9 +1,9 @@use std::collections::HashMap;-use std::sync::{Arc, Mutex};+use std::sync::Arc;use std::time::Duration;use common_kafka_consumer::{- AssignmentEpoch, GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,+ AssignmentEpoch, Offset, Partition, TopicOffsetLedger, TopicPartition,};use lifecycle::Handle;use metrics::{counter, gauge, histogram};@@ -11,11 +11,10 @@ use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};use rdkafka::error::KafkaError;use rdkafka::message::BorrowedMessage;use rdkafka::TopicPartitionList;-use tokio::sync::mpsc;use tokio::time::MissedTickBehavior;use tracing::{error, info, warn};-use crate::batcher::{make_batch_id, Batcher, BatcherObserver, BatcherOutputs, Revoker};+use crate::batcher::{make_batch_id, Batcher, BatcherCompletion, BatcherObserver, BatcherOutputs};use crate::charge_limit::ChargeLimit;use crate::config::Config;use crate::debug_recorder::{record_if, CommitOffset, DebugEventKind, DebugRecorder};@@ -27,62 +26,16 @@ use crate::polling::{debug_partition_offsets, debug_partition_spans, BatchStats, CollectedBatch, ContinuousPolling,InFlightPoll, PartitionDeliveries, Parts, PollLimits, PollingStrategy, Step,};-use crate::rebalance::{partition_error, PartitionHook, RebalanceContext};+use crate::rebalance::{+ partition_error, partition_names, Drain, Event, RebalanceContext, Unassign,+};use crate::settlement::{Commit, GroupSettlement, Settlement};-fn revoke_hook(- revoker: Revoker,- settlement: Arc<GroupSettlement>,- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,-) -> PartitionHook {- Box::new(move |partitions| {- settlement.forget_partitions(partitions);- let pairs: Vec<(String, i32)> = partitions- .iter()- .map(|tp| (tp.topic.clone(), tp.partition))- .collect();- revoker.purge_revoked(&pairs);- revoked_partitions- .lock()- .unwrap()- .extend_from_slice(partitions);- })-}--fn assign_hook(- settlement: Arc<GroupSettlement>,- assignment_epoch: AssignmentEpoch,- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,-) -> PartitionHook {- Box::new(move |partitions| {- // Also covers a loss with no revoke callback (an error rebalance, a fenced member).- settlement.forget_partitions(partitions);- assignment_epoch.bump();- revoked_partitions- .lock()- .unwrap()- .extend_from_slice(partitions);- })-}--fn wire_rebalance_hooks(- context: &RebalanceContext,- revoker: Revoker,- settlement: &Arc<GroupSettlement>,- assignment_epoch: AssignmentEpoch,-) -> Arc<Mutex<Vec<TopicPartition>>> {- let revoked_partitions = Arc::new(Mutex::new(Vec::new()));- context.set_on_revoke(revoke_hook(- revoker,- Arc::clone(settlement),- Arc::clone(&revoked_partitions),- ));- context.set_on_assign(assign_hook(- Arc::clone(settlement),- assignment_epoch,- Arc::clone(&revoked_partitions),- ));- revoked_partitions+fn batcher_partitions(partitions: &[TopicPartition]) -> Vec<(String, i32)> {+ partitions+ .iter()+ .map(|tp| (tp.topic.clone(), tp.partition))+ .collect()}fn commit_tick(commit_interval: Duration) -> Duration {@@ -107,6 +60,7 @@ pub struct IngestionConsumerOptions {pub group_id: String,pub debug_recorder: Option<Arc<DebugRecorder>>,pub commit_interval: Duration,+ pub revoke_drain_timeout: Option<Duration>,pub ledger: Arc<TopicOffsetLedger>,pub partition_budget: BudgetLimits,}@@ -130,9 +84,8 @@ pub struct IngestionConsumer {group_id: String,debug_recorder: Option<Arc<DebugRecorder>>,topic_offset_ledger: Arc<TopicOffsetLedger>,- /// Partitions revoked since the loop last looked, fed by the rebalance- /// callback.- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,+ assignment_epoch: AssignmentEpoch,+ revoke_drain_timeout: Option<Duration>,settlement: Arc<GroupSettlement>,}@@ -154,17 +107,12 @@ impl IngestionConsumer {options.commit_interval,options.partition_budget,));- let revoked_partitions = wire_rebalance_hooks(- consumer.context(),- batcher.revoker(),- &settlement,- transport.assignment_epoch(),- );let consumer = Arc::new(consumer);Self {debug_recorder: options.debug_recorder,topic_offset_ledger,- revoked_partitions,+ assignment_epoch: transport.assignment_epoch(),+ revoke_drain_timeout: options.revoke_drain_timeout,settlement,consumer,batcher,@@ -217,21 +165,17 @@ impl IngestionConsumer {let in_flight_limit = config.in_flight_limit().map_err(|err| anyhow::anyhow!("invalid in-flight limit: {err}"))?;+ let revoke_drain_timeout = config+ .revoke_drain_timeout()+ .map_err(|err| anyhow::anyhow!("invalid revoke drain timeout: {err}"))?;let commit_interval = Duration::from_millis(config.consumer_commit_interval_ms);let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(Arc::clone(&topic_offset_ledger),commit_interval,partition_budget,));- let context = RebalanceContext::new();- let revoked_partitions = wire_rebalance_hooks(- &context,- batcher.revoker(),- &settlement,- transport.assignment_epoch(),- );let consumer: StreamConsumer<RebalanceContext> =- client_config.create_with_context(context)?;+ client_config.create_with_context(RebalanceContext::new())?;consumer.subscribe(&[&config.ingestion_consumer_consume_topic])?;let consumer = Arc::new(consumer);@@ -249,7 +193,8 @@ impl IngestionConsumer {consumer,debug_recorder,topic_offset_ledger,- revoked_partitions,+ assignment_epoch: transport.assignment_epoch(),+ revoke_drain_timeout,settlement,batcher,outputs: Some(outputs),@@ -273,10 +218,7 @@ impl IngestionConsumer {/// First waits until the statically configured workers are ready.pub async fn process(mut self) {let _guard = self.handle.process_scope();- let BatcherOutputs {- completions,- errors,- } = self.outputs.take().expect("process is called once");+ let outputs = self.outputs.take().expect("process is called once");info!("Waiting for workers to be ready");if let Err(err) = self@@ -306,29 +248,44 @@ impl IngestionConsumer {group_id: self.group_id.clone(),};let poller = ContinuousPolling::new(parts, self.in_flight_limit);- self.run(poller, completions, errors).await;+ self.run(poller, outputs).await;if let Err(err) = self.commit_all() {warn!(error = %err, "Could not submit pending frontiers on exit");}info!("Consumer loop stopped");}- async fn run<P: PollingStrategy>(- &self,- mut poller: P,- mut completions: mpsc::UnboundedReceiver<GroupCompletion>,- mut errors: mpsc::UnboundedReceiver<String>,- ) {+ fn rebalance(&self) -> &RebalanceContext {+ self.consumer.context()+ }++ async fn run<P: PollingStrategy>(&self, mut poller: P, outputs: BatcherOutputs) {+ self.rebalance().on_started(self.revoke_drain_timeout);+ self.run_until_stopped(&mut poller, outputs).await;+ let drain = self.rebalance().on_stopped();+ self.apply_rebalance_events(&mut poller);+ if let Err(err) = self.finish_drain(&mut poller, drain, "stopped").await {+ warn!(error = %err, "Could not commit the revoked partitions on exit");+ }+ }++ async fn run_until_stopped<P: PollingStrategy>(&self, poller: &mut P, outputs: BatcherOutputs) {+ let BatcherOutputs {+ mut completions,+ mut errors,+ } = outputs;let mut tick = tokio::time::interval(self.commit_tick);tick.set_missed_tick_behavior(MissedTickBehavior::Delay);let poll_deadline = tokio::time::sleep(Duration::ZERO);tokio::pin!(poll_deadline);+ let drain_deadline = tokio::time::sleep(Duration::ZERO);+ tokio::pin!(drain_deadline);let mut accepting_new_batches = true;loop {let mut submitted = false;while let Some(step) = poller.next_step(tokio::time::Instant::now()) {submitted |= matches!(step, Step::Submit(_));- if let Err(failure) = self.carry_out(step, &mut poller) {+ if let Err(failure) = self.carry_out(step, poller) {match failure {Stopped::Drained => {}Stopped::Commit(err) => self.fail_commit(err),@@ -350,6 +307,12 @@ impl IngestionConsumer {poll_deadline.as_mut().reset(deadline);}}+ let drain_until = self.rebalance().drain_deadline();+ if let Some(until) = drain_until {+ if drain_deadline.deadline() != until {+ drain_deadline.as_mut().reset(until);+ }+ }let reading = poller.reading();let woke = tokio::select! {_ = self.handle.shutdown_recv(), if accepting_new_batches => Woke::Shutdown,@@ -357,22 +320,30 @@ impl IngestionConsumer {failure = errors.recv() => Woke::Error(failure),_ = tick.tick() => Woke::Tick,_ = &mut poll_deadline, if deadline.is_some() => Woke::Deadline,+ _ = &mut drain_deadline, if drain_until.is_some() => Woke::DrainTimedOut,+ _ = self.rebalance().wait_for_rebalance() => Woke::Rebalanced,received = self.consumer.recv(), if reading => Woke::Received(received),};- // The rebalance callbacks run inside `recv`, also inside one the- // select dropped, so the poller sees each revoke before it acts- // on anything that woke the loop.- self.hand_over_revokes(&mut poller);+ // The rebalance callbacks run inside `recv`, also in a `recv` that+ // the select dropped. Their events apply before anything that woke+ // the loop.+ self.apply_rebalance_events(poller);let failure = match woke {Woke::Shutdown => {info!("Shutdown signal received, draining in-flight batches");accepting_new_batches = false;self.batcher.begin_shutdown();- poller.shutdown();+ poller.on_shutdown();None}- Woke::Completion(Some(completion)) => poller.completed(&completion).err(),+ Woke::Completion(Some(BatcherCompletion::Group(completion))) => {+ poller.on_completed(&completion).err()+ }+ Woke::Completion(Some(BatcherCompletion::Drained(revoke))) => {+ let drain = self.rebalance().on_drained(revoke);+ self.finish_drain(poller, drain, "drained").await.err()+ }Woke::Completion(None) => {Some(anyhow::anyhow!("batcher completion channel closed"))}@@ -380,12 +351,18 @@ impl IngestionConsumer {Woke::Error(None) => Some(anyhow::anyhow!("batcher error channel closed")),Woke::Tick => {self.handle.report_healthy();- poller.ticked();+ poller.on_ticked();None}- Woke::Deadline => None,+ Woke::Deadline | Woke::Rebalanced => None,+ Woke::DrainTimedOut => {+ let drain = self+ .rebalance()+ .on_drain_timed_out(tokio::time::Instant::now());+ self.finish_drain(poller, drain, "timed_out").await.err()+ }Woke::Received(Ok(message)) => {- poller.received(&message, tokio::time::Instant::now());+ poller.on_received(&message, tokio::time::Instant::now());None}Woke::Received(Err(err)) => self.recv_failed(err).err(),@@ -401,7 +378,7 @@ impl IngestionConsumer {match step {Step::Submit(collected) => {let poll = self.submit_poll(collected).map_err(Stopped::Batch)?;- poller.submitted(poll);+ poller.on_submitted(poll);self.handle.report_healthy();Ok(())}@@ -410,13 +387,90 @@ impl IngestionConsumer {}}- fn hand_over_revokes<P: PollingStrategy>(&self, poller: &mut P) {- let revoked = std::mem::take(&mut *self.revoked_partitions.lock().unwrap());- if !revoked.is_empty() {- poller.revoked(&revoked);+ fn apply_rebalance_events<P: PollingStrategy>(&self, poller: &mut P) {+ let events = self.rebalance().take_events();+ for event in events {+ match event {+ Event::Assigned(assigned) => {+ self.settlement.on_assigned(&assigned);+ self.assignment_epoch.bump();+ poller.on_assigned(&assigned);+ }+ Event::Revoked {+ revoke,+ partitions: revoked,+ unassign,+ } => {+ self.settlement.on_revoked(&revoked);+ self.batcher.revoke(revoke, &batcher_partitions(&revoked));+ poller.on_revoked(&revoked);+ unassign.apply(&*self.consumer);+ }+ Event::Draining {+ revoke,+ partitions: draining,+ } => {+ self.batcher.revoke(revoke, &batcher_partitions(&draining));+ poller.on_drain_started(&draining);+ }+ }}}+ async fn finish_drain<P: PollingStrategy>(+ &self,+ poller: &mut P,+ drain: Option<Drain>,+ outcome: &'static str,+ ) -> anyhow::Result<()> {+ let Some(Drain {+ partitions: drained,+ unassign,+ started,+ timeout,+ ..+ }) = drain+ else {+ return Ok(());+ };+ let unassign_on_drop = UnassignOnDrop {+ consumer: Arc::clone(&self.consumer),+ unassign: Some(unassign),+ };+ let committed = self.commit_drained(poller, &drained, timeout).await;+ self.settlement.on_revoked(&drained);+ drop(unassign_on_drop);+ let elapsed = started.elapsed();+ counter!("ingestion_consumer_revoke_drains_total", "outcome" => outcome).increment(1);+ histogram!("ingestion_consumer_revoke_drain_seconds").record(elapsed.as_secs_f64());+ info!(+ topic_partitions = ?partition_names(&drained),+ outcome,+ elapsed_ms = elapsed.as_millis() as u64,+ "Rebalance: drained partitions unassigned"+ );+ committed+ }++ async fn commit_drained<P: PollingStrategy>(+ &self,+ poller: &mut P,+ partitions: &[TopicPartition],+ timeout: Duration,+ ) -> anyhow::Result<()> {+ let Some(commit) = poller.on_drain_finished(partitions) else {+ return Ok(());+ };+ for topic_partition in &commit.resume {+ self.resume_partition(topic_partition);+ }+ if let Err(err) = self.commit_offsets_sync(&commit.offsets, timeout).await {+ counter!("ingestion_consumer_revoke_commit_errors_total").increment(1);+ warn!(error = %err, "Could not commit the revoked partitions before their release");+ }+ Ok(())+ }+fn recv_failed(&self, err: KafkaError) -> anyhow::Result<()> {warn!(error = %err, "Kafka recv error");counter!("ingestion_consumer_kafka_errors_total").increment(1);@@ -433,6 +487,34 @@ impl IngestionConsumer {/// Submit each partition's next-to-read offset to Kafka, asynchronously.fn commit_offsets(&self, offsets: &HashMap<TopicPartition, Offset>) -> anyhow::Result<()> {+ let tpl = self.commit_list(offsets)?;+ self.consumer.commit(&tpl, CommitMode::Async)?;+ self.record_commit(offsets);+ Ok(())+ }++ /// A commit that takes longer than `timeout` can still land after the+ /// unassign.+ async fn commit_offsets_sync(+ &self,+ offsets: &HashMap<TopicPartition, Offset>,+ timeout: Duration,+ ) -> anyhow::Result<()> {+ let tpl = self.commit_list(offsets)?;+ let consumer = Arc::clone(&self.consumer);+ let commit = tokio::task::spawn_blocking(move || consumer.commit(&tpl, CommitMode::Sync));+ match tokio::time::timeout(timeout, commit).await {+ Ok(joined) => joined??,+ Err(_) => anyhow::bail!("no answer within {timeout:?}"),+ }+ self.record_commit(offsets);+ Ok(())+ }++ fn commit_list(+ &self,+ offsets: &HashMap<TopicPartition, Offset>,+ ) -> anyhow::Result<TopicPartitionList> {let mut tpl = TopicPartitionList::new();for (topic_partition, next_to_read) in offsets {tpl.add_partition_offset(@@ -449,7 +531,10 @@ impl IngestionConsumer {).set(next_to_read.0 as f64);}- self.consumer.commit(&tpl, CommitMode::Async)?;+ Ok(tpl)+ }++ fn record_commit(&self, offsets: &HashMap<TopicPartition, Offset>) {counter!("ingestion_consumer_offset_commits_total").increment(1);record_if(&self.debug_recorder, || DebugEventKind::CommitSubmitted {offsets: offsets@@ -461,7 +546,6 @@ impl IngestionConsumer {}).collect(),});- Ok(())}fn submit_poll(&self, collected: CollectedBatch) -> anyhow::Result<InFlightPoll> {@@ -601,12 +685,37 @@ impl IngestionConsumer {}}+struct UnassignOnDrop {+ consumer: Arc<StreamConsumer<RebalanceContext>>,+ unassign: Option<Unassign>,+}++impl Drop for UnassignOnDrop {+ fn drop(&mut self) {+ if let Some(unassign) = self.unassign.take() {+ unassign.apply(&*self.consumer);+ }+ }+}++/// A dropped task never reaches the end of `run`. The rdkafka close that+/// follows waits for every unassign the loop still owes.+impl Drop for IngestionConsumer {+ fn drop(&mut self) {+ for unassign in self.rebalance().on_dropped() {+ unassign.apply(&*self.consumer);+ }+ }+}+enum Woke<'a> {Shutdown,- Completion(Option<GroupCompletion>),+ Completion(Option<BatcherCompletion>),Error(Option<String>),Tick,Deadline,+ DrainTimedOut,+ Rebalanced,Received(Result<BorrowedMessage<'a>, KafkaError>),}@@ -680,11 +789,34 @@ fn emit_poll_stats(#[cfg(test)]mod tests {use super::*;- use std::num::NonZeroU64;- use common_kafka_consumer::{Charge, Offset as MessageOffset};+ use rdkafka::config::ClientConfig;++ #[tokio::test]+ async fn a_drain_commit_cancelled_midway_still_unassigns() {+ let consumer: StreamConsumer<RebalanceContext> = ClientConfig::new()+ .set("bootstrap.servers", "127.0.0.1:1")+ .set("group.id", "test")+ .create_with_context(RebalanceContext::new())+ .expect("a client needs no broker to be built");+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition("events", 0);+ consumer.assign(&tpl).expect("an assign needs no broker");+ let consumer = Arc::new(consumer);+ let unassign_on_drop = UnassignOnDrop {+ consumer: Arc::clone(&consumer),+ unassign: Some(Unassign::new(tpl, false)),+ };++ let commit = tokio::spawn(async move {+ let _unassign_on_drop = unassign_on_drop;+ std::future::pending::<()>().await;+ });+ commit.abort();+ let _ = commit.await;- use crate::settlement::{GroupSettlement, Pause};+ assert_eq!(consumer.assignment().expect("assignment").count(), 0);+ }#[rstest::rstest]#[case::below_a_second(200, 200)]@@ -699,54 +831,4 @@ mod tests {Duration::from_millis(expected_ms));}-- #[test]- fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {- let ledger = Arc::new(TopicOffsetLedger::new());- let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(- Arc::clone(&ledger),- Duration::from_secs(3600),- BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),- ));- let epoch = AssignmentEpoch::new();- let revoked_partitions = Arc::new(Mutex::new(Vec::new()));- let on_assign = assign_hook(- Arc::clone(&settlement),- epoch.clone(),- Arc::clone(&revoked_partitions),- );- let p0 = TopicPartition::new("events", 0);- let one_event = [(- MessageOffset(10),- Charge {- events: 1,- bytes: 0,- },- )];- assert_eq!(- settlement- .charge(&p0, 0, &one_event)- .map(|charged| charged.pause),- Ok(Some(Pause))- );- settlement.settle_run(&p0, 0, &[MessageOffset(10)]).unwrap();-- on_assign(std::slice::from_ref(&p0));-- assert!(settlement.take_all().is_none());- assert_eq!(ledger.generation(&p0), 1);- assert_eq!(epoch.current(), 2);- assert_eq!(- *revoked_partitions.lock().unwrap(),- vec![p0.clone()],- "the poller drops what it buffered from the lost assignment"- );- assert_eq!(- settlement- .charge(&p0, 1, &one_event)- .map(|charged| charged.pause),- Ok(Some(Pause)),- "the regained partition starts unpaused"- );- }}diff --git a/rust/ingestion-consumer/src/main.rs b/rust/ingestion-consumer/src/main.rsindex c036f04e4977..ca87894f81e0 100644--- a/rust/ingestion-consumer/src/main.rs+++ b/rust/ingestion-consumer/src/main.rs@@ -22,7 +22,7 @@ use tracing_subscriber::{EnvFilter, Layer};use ingestion_consumer::aperture;use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;use ingestion_consumer::batcher::{Batcher, BatcherObserver};-use ingestion_consumer::config::Config;+use ingestion_consumer::config::{Config, GRACEFUL_SHUTDOWN, LIVENESS_DEADLINE};use ingestion_consumer::consumer::IngestionConsumer;use ingestion_consumer::debug_recorder::{BatcherLoad, DebugLoad, DebugRecorder, DebugState, GovernorLoad, RoutingDebug, WorkerStatus,@@ -56,13 +56,19 @@ fn main() -> Result<()> {config.in_flight_limit().map_err(|err| anyhow::anyhow!("invalid in-flight limit: {err}"))?;+ config+ .revoke_drain_timeout()+ .map_err(|err| anyhow::anyhow!("invalid revoke drain timeout: {err}"))?;let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().context("Failed to build tokio runtime")?;- runtime.block_on(async_main(config))+ let result = runtime.block_on(async_main(config));+ // A blocking commit that outlived its timeout must not hold the exit.+ runtime.shutdown_timeout(Duration::from_secs(5));+ result}async fn async_main(config: Config) -> Result<()> {@@ -116,8 +122,8 @@ async fn async_main(config: Config) -> Result<()> {let consumer_handle = manager.register("consumer",ComponentOptions::new()- .with_graceful_shutdown(Duration::from_secs(60))- .with_liveness_deadline(Duration::from_secs(60))+ .with_graceful_shutdown(GRACEFUL_SHUTDOWN)+ .with_liveness_deadline(LIVENESS_DEADLINE).with_stall_threshold(3),);diff --git a/rust/ingestion-consumer/src/polling/continuous.rs b/rust/ingestion-consumer/src/polling/continuous.rsindex edb7d7ad1be2..97411e13300a 100644--- a/rust/ingestion-consumer/src/polling/continuous.rs+++ b/rust/ingestion-consumer/src/polling/continuous.rs@@ -55,11 +55,11 @@ impl PollingStrategy for ContinuousPolling {self.collector.deadline()}- fn received<M: Message>(&mut self, message: &M, now: Instant) {+ fn on_received<M: Message>(&mut self, message: &M, now: Instant) {self.collector.add(message, now);}- fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {+ fn on_completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {if (completion.accepted as usize) < completion.offsets.len() {anyhow::bail!("accepted {}/{} messages — not committing offsets",@@ -84,26 +84,40 @@ impl PollingStrategy for ContinuousPolling {Ok(())}- fn ticked(&mut self) {+ fn on_ticked(&mut self) {self.commit_due = true;}- fn revoked(&mut self, topic_partitions: &[TopicPartition]) {- let building = self.collector.remove_partitions(topic_partitions) as u64;- self.uncommitted.revoked(topic_partitions);- self.processed.revoked(topic_partitions);- report_revoked(building);+ fn on_assigned(&mut self, topic_partitions: &[TopicPartition]) {+ self.forget_partitions(topic_partitions);+ }++ fn on_drain_started(&mut self, topic_partitions: &[TopicPartition]) {+ report_revoked(self.collector.remove_partitions(topic_partitions) as u64);+ }++ fn on_drain_finished(&mut self, topic_partitions: &[TopicPartition]) -> Option<Commit> {+ let commit = self+ .settlement+ .take_all()+ .map(|commit| self.record_commit(commit));+ self.on_revoked(topic_partitions);+ commit+ }++ fn on_revoked(&mut self, topic_partitions: &[TopicPartition]) {+ self.forget_partitions(topic_partitions);}- fn shutdown(&mut self) {+ fn on_shutdown(&mut self) {self.accepting_new_batches = false;self.commit_due = true;self.collector.discard();}- fn submitted(&mut self, poll: InFlightPoll) {- self.uncommitted.charged(&poll);- self.processed.submitted(&poll);+ fn on_submitted(&mut self, poll: InFlightPoll) {+ self.uncommitted.record_poll(&poll);+ self.processed.record_poll(&poll);}fn next_step(&mut self, now: Instant) -> Option<Step> {@@ -114,9 +128,7 @@ impl PollingStrategy for ContinuousPolling {self.settlement.take_all()};if let Some(commit) = commit {- self.uncommitted.committed(&commit);- self.processed.committed(&commit);- return Some(Step::Commit(commit));+ return Some(Step::Commit(self.record_commit(commit)));}}if let Some(collected) = self.collector.take_ready(now) {@@ -126,6 +138,21 @@ impl PollingStrategy for ContinuousPolling {}}+impl ContinuousPolling {+ fn record_commit(&mut self, commit: Commit) -> Commit {+ self.uncommitted.record_commit(&commit);+ self.processed.record_commit(&commit);+ commit+ }++ fn forget_partitions(&mut self, topic_partitions: &[TopicPartition]) {+ let building = self.collector.remove_partitions(topic_partitions) as u64;+ self.uncommitted.forget(topic_partitions);+ self.processed.forget(topic_partitions);+ report_revoked(building);+ }+}+struct ProcessedTimestamps {group_id: String,partitions: HashMap<TopicPartition, VecDeque<SubmittedSlice>>,@@ -144,7 +171,7 @@ impl ProcessedTimestamps {}}- fn submitted(&mut self, poll: &InFlightPoll) {+ fn record_poll(&mut self, poll: &InFlightPoll) {for (topic_partition, deliveries) in &poll.partitions {self.partitions.entry(topic_partition.clone())@@ -156,7 +183,7 @@ impl ProcessedTimestamps {}}- fn committed(&mut self, commit: &Commit) {+ fn record_commit(&mut self, commit: &Commit) {for (topic_partition, next_to_read) in &commit.offsets {let Some(polls) = self.partitions.get_mut(topic_partition) else {continue;@@ -182,8 +209,8 @@ impl ProcessedTimestamps {}}- fn revoked(&mut self, revoked: &[TopicPartition]) {- for topic_partition in revoked {+ fn forget(&mut self, topic_partitions: &[TopicPartition]) {+ for topic_partition in topic_partitions {self.partitions.remove(topic_partition);}}@@ -205,14 +232,14 @@ impl UncommittedCharges {self.total == Charge::ZERO}- /// `None` once the partition is revoked, until its next charge.+ /// `None` once the partition is forgotten, until its next charge.fn topic_partition(&self, partition: Partition) -> Option<&TopicPartition> {self.partitions.get(&partition).map(|(topic_partition, _)| topic_partition)}- fn charged(&mut self, poll: &InFlightPoll) {+ fn record_poll(&mut self, poll: &InFlightPoll) {for (topic_partition, &charge) in &poll.uncommitted {let previous = self.partitions.insert(Partition(topic_partition.partition),@@ -222,7 +249,7 @@ impl UncommittedCharges {}}- fn committed(&mut self, commit: &Commit) {+ fn record_commit(&mut self, commit: &Commit) {for (topic_partition, &charge) in &commit.uncommitted {if let Some((_, partition)) = self.partitions@@ -234,8 +261,8 @@ impl UncommittedCharges {}}- fn revoked(&mut self, revoked: &[TopicPartition]) {- for topic_partition in revoked {+ fn forget(&mut self, topic_partitions: &[TopicPartition]) {+ for topic_partition in topic_partitions {let previous = self.partitions.remove(&Partition(topic_partition.partition));@@ -296,7 +323,7 @@ mod tests {harness.receive(0, &[10, 11]);harness.receive(1, &[20]);- harness.poller.revoked(&[p(0)]);+ harness.poller.on_revoked(&[p(0)]);harness.receive(0, &[10]);harness.after_the_batch_timeout();@@ -339,7 +366,7 @@ mod tests {harness.submit();harness.complete_under(before, 0, &[0, 1]);- harness.poller.ticked();+ harness.poller.on_ticked();assert!(harness.step().is_none(),"the old assignment's completion is stale"@@ -350,6 +377,66 @@ mod tests {assert_eq!(processed(), 2);}+ #[test]+ fn a_started_drain_drops_the_poll_in_progress_and_settles_what_was_submitted() {+ let mut harness = continuous(1_000);+ harness.receive(0, &[0, 1]);+ harness.after_the_batch_timeout();+ harness.submit();+ harness.receive(0, &[2]);++ harness.poller.on_drain_started(&[p(0)]);+ harness.after_the_batch_timeout();+ assert!(+ harness.step().is_none(),+ "offset 2 was never charged, so the new owner reads it"+ );++ harness.complete(0, &[0, 1]);+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));+ }++ #[test]+ fn a_finished_drain_commits_up_to_the_first_unprocessed_offset() {+ let mut harness = continuous(1_000);+ harness.receive(0, &[0, 1, 2]);+ harness.after_the_batch_timeout();+ harness.submit();++ harness.poller.on_drain_started(&[p(0)]);+ harness.complete(0, &[0, 2]);++ assert_eq!(+ harness.finish_drain(0),+ HashMap::from([(p(0), Offset(1))]),+ "offset 1 came back unfinished"+ );+ }++ #[test]+ fn a_finished_drain_frees_the_partitions_charge_and_ignores_its_late_completions() {+ let mut harness = continuous(3);+ harness.receive(0, &[0, 1, 2]);+ harness.after_the_batch_timeout();+ harness.submit();+ assert!(!harness.poller.reading());+ let drained_generation = harness.ledger.generation(&p(0));++ harness.poller.on_drain_started(&[p(0)]);+ assert_eq!(harness.finish_drain(0), HashMap::new());+ assert!(+ harness.poller.reading(),+ "a drained partition holds no charge"+ );++ harness.complete_under(drained_generation, 0, &[0, 1, 2]);+ harness.poller.on_ticked();+ assert!(+ harness.step().is_none(),+ "a late completion commits nothing"+ );+ }+#[test]fn a_revoke_frees_the_partitions_charge() {let mut harness = continuous(3);@@ -414,7 +501,7 @@ mod tests {let error = harness.poller- .completed(&short)+ .on_completed(&short).expect_err("a dropped message must not commit");assert!(error.to_string().contains("accepted 1/2"), "{error}");}@@ -458,7 +545,7 @@ mod tests {harness.submit();harness.receive(0, &[1]);- harness.poller.shutdown();+ harness.poller.on_shutdown();assert!(!harness.poller.reading());assert_eq!(harness.poller.deadline(), None);assert!(harness.step().is_none(), "offset 0 is still in flight");diff --git a/rust/ingestion-consumer/src/polling/harness.rs b/rust/ingestion-consumer/src/polling/harness.rsindex bd4e7bee210d..94f4cb2bd20d 100644--- a/rust/ingestion-consumer/src/polling/harness.rs+++ b/rust/ingestion-consumer/src/polling/harness.rs@@ -24,7 +24,7 @@ pub(super) struct Harness<P> {impl<P: PollingStrategy> Harness<P> {pub(super) fn new(make: impl FnOnce(Parts) -> P) -> Self {let ledger = Arc::new(TopicOffsetLedger::new());- let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(+ let settlement = Arc::new(GroupSettlement::new(Arc::clone(&ledger),Duration::ZERO,BudgetLimits::unset(),@@ -49,7 +49,7 @@ impl<P: PollingStrategy> Harness<P> {pub(super) fn receive(&mut self, partition: i32, offsets: &[i64]) {for &offset in offsets {self.poller- .received(&message(partition, offset, 10), self.now);+ .on_received(&message(partition, offset, 10), self.now);}}@@ -84,7 +84,7 @@ impl<P: PollingStrategy> Harness<P> {)}).collect();- self.poller.submitted(InFlightPoll {+ self.poller.on_submitted(InFlightPoll {partitions,uncommitted,});@@ -98,13 +98,23 @@ impl<P: PollingStrategy> Harness<P> {pub(super) fn complete_under(&mut self, generation: u64, partition: i32, offsets: &[i64]) {self.poller- .completed(&completion(generation, partition, offsets))+ .on_completed(&completion(generation, partition, offsets)).expect("the completion applies");}pub(super) fn revoke(&mut self, partition: i32) {- self.settlement.forget_partitions(&[p(partition)]);- self.poller.revoked(&[p(partition)]);+ self.settlement.on_revoked(&[p(partition)]);+ self.poller.on_revoked(&[p(partition)]);+ }++ pub(super) fn finish_drain(&mut self, partition: i32) -> HashMap<TopicPartition, Offset> {+ let committed = self+ .poller+ .on_drain_finished(&[p(partition)])+ .map(|commit| commit.offsets)+ .unwrap_or_default();+ self.settlement.on_revoked(&[p(partition)]);+ committed}pub(super) fn commit(&mut self) -> HashMap<TopicPartition, Offset> {diff --git a/rust/ingestion-consumer/src/polling/mod.rs b/rust/ingestion-consumer/src/polling/mod.rsindex feb998cd9979..bfe2becf71b8 100644--- a/rust/ingestion-consumer/src/polling/mod.rs+++ b/rust/ingestion-consumer/src/polling/mod.rs@@ -47,22 +47,29 @@ pub(crate) trait PollingStrategy {fn deadline(&self) -> Option<Instant>;- fn received<M: Message>(&mut self, message: &M, now: Instant);+ fn on_received<M: Message>(&mut self, message: &M, now: Instant);- fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()>;+ fn on_completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()>;- fn ticked(&mut self);+ fn on_ticked(&mut self);++ fn on_assigned(&mut self, topic_partitions: &[TopicPartition]);++ fn on_drain_started(&mut self, topic_partitions: &[TopicPartition]);++ /// The commit holds every partition's frontier, not only the drained ones.+ fn on_drain_finished(&mut self, topic_partitions: &[TopicPartition]) -> Option<Commit>;/// Every message received before the revoke belongs to the old/// assignment: the batcher purges it, or Kafka delivers it again from the/// last commit.- fn revoked(&mut self, topic_partitions: &[TopicPartition]);+ fn on_revoked(&mut self, topic_partitions: &[TopicPartition]);/// The poll in progress is not charged yet, so it is dropped and replays/// from the last commit.- fn shutdown(&mut self);+ fn on_shutdown(&mut self);- fn submitted(&mut self, poll: InFlightPoll);+ fn on_submitted(&mut self, poll: InFlightPoll);fn next_step(&mut self, now: Instant) -> Option<Step>;}diff --git a/rust/ingestion-consumer/src/rebalance.rs b/rust/ingestion-consumer/src/rebalance.rsdeleted file mode 100644index 76d0e135aa1e..000000000000--- a/rust/ingestion-consumer/src/rebalance.rs+++ /dev/null@@ -1,190 +0,0 @@-//! The consumer's rdkafka context: rebalance callbacks into hooks, plus rebalance counters and client statistics.--use std::sync::OnceLock;--use common_kafka_consumer::TopicPartition;-use metrics::counter;-use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext, Rebalance};-use rdkafka::error::KafkaResult;-use rdkafka::{ClientContext, Statistics, TopicPartitionList};-use tracing::{error, info, warn};--pub type PartitionHook = Box<dyn Fn(&[TopicPartition]) + Send + Sync>;--/// Hooks run inside the callback, before the next poll delivers the new assignment.-#[derive(Default)]-pub struct RebalanceContext {- on_revoke: OnceLock<PartitionHook>,- on_assign: OnceLock<PartitionHook>,-}--impl RebalanceContext {- pub fn new() -> Self {- Self::default()- }-- /// Settable after construction because the rdkafka consumer only hands out `&self`.- pub fn set_on_revoke(&self, hook: PartitionHook) {- let _ = self.on_revoke.set(hook);- }-- pub fn set_on_assign(&self, hook: PartitionHook) {- let _ = self.on_assign.set(hook);- }-}--fn topic_partitions(tpl: &TopicPartitionList) -> Vec<TopicPartition> {- let mut partitions: Vec<TopicPartition> = tpl- .elements()- .iter()- .map(|element| TopicPartition::new(element.topic(), element.partition()))- .collect();- partitions.sort_unstable_by(|a, b| (&a.topic, a.partition).cmp(&(&b.topic, b.partition)));- partitions-}--/// librdkafka's pause and resume return success when they cannot change a-/// partition. They set the error on that partition's list element.-pub(crate) fn partition_error(tpl: &TopicPartitionList) -> KafkaResult<()> {- tpl.elements()- .iter()- .try_for_each(|element| element.error())-}--fn partition_names(partitions: &[TopicPartition]) -> Vec<String> {- partitions- .iter()- .map(|tp| format!("{}:{}", tp.topic, tp.partition))- .collect()-}--impl ClientContext for RebalanceContext {- /// Fired on a librdkafka thread every `statistics.interval.ms`; disabled- /// when that is 0.- fn stats(&self, stats: Statistics) {- crate::kafka_stats::export(&stats);- }-}--impl ConsumerContext for RebalanceContext {- fn pre_rebalance(&self, _consumer: &BaseConsumer<Self>, rebalance: &Rebalance) {- match rebalance {- Rebalance::Revoke(tpl) => {- counter!("ingestion_consumer_rebalances_total", "event" => "revoke").increment(1);- let partitions = topic_partitions(tpl);- info!(- partitions = partitions.len(),- topic_partitions = ?partition_names(&partitions),- "Rebalance: partitions revoked"- );- if let Some(hook) = self.on_revoke.get() {- hook(&partitions);- }- }- Rebalance::Assign(_) => {}- Rebalance::Error(err) => {- counter!("ingestion_consumer_rebalances_total", "event" => "error").increment(1);- warn!(error = %err, "Rebalance error");- }- }- }-- fn post_rebalance(&self, consumer: &BaseConsumer<Self>, rebalance: &Rebalance) {- if let Rebalance::Assign(tpl) = rebalance {- counter!("ingestion_consumer_rebalances_total", "event" => "assign").increment(1);- let partitions = topic_partitions(tpl);- info!(- partitions = partitions.len(),- topic_partitions = ?partition_names(&partitions),- "Rebalance: partitions assigned"- );- // librdkafka keeps a partition's pause across a revoke and a- // reassignment, and the assign hook forgets the budget that paused it.- if let Err(err) = consumer.resume(tpl).and_then(|()| partition_error(tpl)) {- error!(error = %err, "Could not resume the assigned partitions");- }- if let Some(hook) = self.on_assign.get() {- hook(&partitions);- }- }- }-- // NOTE: `ConsumerContext::commit_callback` is deliberately not implemented.- // librdkafka only propagates a commit result to the application when a- // conf-level `offset_commit_cb` is registered (rust-rdkafka never does) or- // when the commit is synchronous (a replyq is attached); manual async- // commits silently drop their result (`rd_kafka_cgrp_propagate_commit_result`).-}--#[cfg(test)]-mod tests {- use std::sync::{Arc, Mutex};-- use rdkafka::config::ClientConfig;- use rdkafka::consumer::Consumer;-- use super::*;-- fn recording(seen: &Arc<Mutex<Vec<TopicPartition>>>) -> PartitionHook {- let seen = Arc::clone(seen);- Box::new(move |partitions| seen.lock().unwrap().extend_from_slice(partitions))- }-- #[test]- fn partitions_sort_by_topic_then_partition_number() {- let mut tpl = TopicPartitionList::new();- tpl.add_partition("overflow", 2);- tpl.add_partition("events", 10);- tpl.add_partition("events", 9);-- assert_eq!(- partition_names(&topic_partitions(&tpl)),- vec!["events:9", "events:10", "overflow:2"]- );- }-- #[test]- fn each_callback_reaches_its_hook_with_the_partitions() {- let revoked = Arc::new(Mutex::new(Vec::new()));- let assigned = Arc::new(Mutex::new(Vec::new()));- let context = RebalanceContext::new();- context.set_on_revoke(recording(&revoked));- context.set_on_assign(recording(&assigned));- let consumer: BaseConsumer<RebalanceContext> = ClientConfig::new()- .set("bootstrap.servers", "127.0.0.1:1")- .set("group.id", "test")- .create_with_context(context)- .expect("a client needs no broker to be built");- let mut tpl = TopicPartitionList::new();- tpl.add_partition("events", 1);- tpl.add_partition("events", 0);-- consumer- .context()- .pre_rebalance(&consumer, &Rebalance::Revoke(&tpl));- consumer- .context()- .post_rebalance(&consumer, &Rebalance::Assign(&tpl));-- let expected = vec![- TopicPartition::new("events", 0),- TopicPartition::new("events", 1),- ];- assert_eq!(*revoked.lock().unwrap(), expected);- assert_eq!(*assigned.lock().unwrap(), expected);- }-- #[test]- fn pausing_an_unassigned_partition_is_an_error() {- let consumer: BaseConsumer = ClientConfig::new()- .set("bootstrap.servers", "127.0.0.1:1")- .set("group.id", "test")- .create()- .expect("a client needs no broker to be built");- let mut tpl = TopicPartitionList::new();- tpl.add_partition("events", 0);-- assert!(consumer.pause(&tpl).is_ok());- assert!(partition_error(&tpl).is_err());- }-}diff --git a/rust/ingestion-consumer/src/rebalance/context.rs b/rust/ingestion-consumer/src/rebalance/context.rsnew file mode 100644index 000000000000..3f46b11afcda--- /dev/null+++ b/rust/ingestion-consumer/src/rebalance/context.rs@@ -0,0 +1,262 @@+//! The consumer's rdkafka context: rebalance callbacks into [`RebalanceState`], plus rebalance counters and client statistics.++use std::sync::{Mutex, MutexGuard};+use std::time::Duration;++use common_kafka_consumer::TopicPartition;+use metrics::counter;+use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext, RebalanceProtocol};+use rdkafka::error::{KafkaResult, RDKafkaErrorCode};+use rdkafka::types::RDKafkaRespErr;+use rdkafka::{ClientContext, Statistics, TopicPartitionList};+use tokio::sync::Notify;+use tokio::time::Instant;+use tracing::{error, info, warn};++use super::state::{Drain, Event, RebalanceState, Unassign};+use crate::batcher::RevokeId;++#[derive(Default)]+pub struct RebalanceContext {+ rebalance_state: Mutex<RebalanceState>,+ rebalanced: Notify,+}++impl RebalanceContext {+ pub fn new() -> Self {+ Self::default()+ }++ pub(crate) fn on_started(&self, drain_timeout: Option<Duration>) {+ self.rebalance_state().on_started(drain_timeout);+ }++ pub(crate) fn on_stopped(&self) -> Option<Drain> {+ self.rebalance_state().on_stopped()+ }++ pub(crate) fn on_dropped(&self) -> Vec<Unassign> {+ self.rebalance_state().on_dropped()+ }++ pub(crate) fn on_drained(&self, revoke: RevokeId) -> Option<Drain> {+ self.rebalance_state().on_drained(revoke)+ }++ pub(crate) fn on_drain_timed_out(&self, now: Instant) -> Option<Drain> {+ self.rebalance_state().on_drain_timed_out(now)+ }++ pub(crate) fn take_events(&self) -> Vec<Event> {+ self.rebalance_state().take_events()+ }++ pub(crate) fn drain_deadline(&self) -> Option<Instant> {+ self.rebalance_state().drain_deadline()+ }++ pub(crate) async fn wait_for_rebalance(&self) {+ self.rebalanced.notified().await;+ }++ fn rebalance_state(&self) -> MutexGuard<'_, RebalanceState> {+ self.rebalance_state.lock().unwrap()+ }++ fn on_assigned(&self, consumer: &BaseConsumer<Self>, tpl: &TopicPartitionList) {+ counter!("ingestion_consumer_rebalances_total", "event" => "assign").increment(1);+ let partitions = topic_partitions(tpl);+ info!(+ partitions = partitions.len(),+ topic_partitions = ?partition_names(&partitions),+ "Rebalance: partitions assigned"+ );+ let assigned = if cooperative(consumer) {+ consumer.incremental_assign(tpl)+ } else {+ consumer.assign(tpl)+ };+ if let Err(err) = assigned {+ error!(error = %err, "Could not assign the partitions");+ }+ // librdkafka keeps a partition's pause across a revoke and a+ // reassignment, and the consumer forgets the budget that paused it.+ if let Err(err) = consumer.resume(tpl).and_then(|()| partition_error(tpl)) {+ error!(error = %err, "Could not resume the assigned partitions");+ }+ self.rebalance_state().on_assigned(partitions);+ }++ fn on_revoked(&self, consumer: &BaseConsumer<Self>, tpl: &TopicPartitionList) {+ counter!("ingestion_consumer_rebalances_total", "event" => "revoke").increment(1);+ let partitions = topic_partitions(tpl);+ let names = partition_names(&partitions);+ info!(+ partitions = names.len(),+ topic_partitions = ?names,+ "Rebalance: partitions revoked"+ );+ let unassign = self.rebalance_state().on_revoked(+ partitions,+ Unassign::new(tpl.clone(), cooperative(consumer)),+ consumer.assignment_lost(),+ Instant::now(),+ );+ if let Some(unassign) = unassign {+ unassign.apply(consumer);+ }+ }+}++fn cooperative<C: ConsumerContext>(consumer: &BaseConsumer<C>) -> bool {+ matches!(+ consumer.rebalance_protocol(),+ RebalanceProtocol::Cooperative+ )+}++fn topic_partitions(tpl: &TopicPartitionList) -> Vec<TopicPartition> {+ let mut partitions: Vec<TopicPartition> = tpl+ .elements()+ .iter()+ .map(|element| TopicPartition::new(element.topic(), element.partition()))+ .collect();+ partitions.sort_unstable_by(|a, b| (&a.topic, a.partition).cmp(&(&b.topic, b.partition)));+ partitions+}++/// librdkafka's pause and resume return success when they cannot change a+/// partition. They set the error on that partition's list element.+pub(crate) fn partition_error(tpl: &TopicPartitionList) -> KafkaResult<()> {+ tpl.elements()+ .iter()+ .try_for_each(|element| element.error())+}++pub(crate) fn partition_names(partitions: &[TopicPartition]) -> Vec<String> {+ partitions+ .iter()+ .map(|tp| format!("{}:{}", tp.topic, tp.partition))+ .collect()+}++impl ClientContext for RebalanceContext {+ /// Fired on a librdkafka thread every `statistics.interval.ms`; disabled+ /// when that is 0.+ fn stats(&self, stats: Statistics) {+ crate::kafka_stats::export(&stats);+ }+}++impl ConsumerContext for RebalanceContext {+ fn rebalance(+ &self,+ consumer: &BaseConsumer<Self>,+ err: RDKafkaRespErr,+ tpl: &mut TopicPartitionList,+ ) {+ match err {+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS => self.on_assigned(consumer, tpl),+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS => self.on_revoked(consumer, tpl),+ // librdkafka only calls back with assign or revoke. This arm is a+ // fallback for another code, and the unassign keeps the group from+ // waiting on this member.+ _ => {+ warn!(error = %RDKafkaErrorCode::from(err), "Rebalance error");+ Unassign::new(tpl.clone(), cooperative(consumer)).apply(consumer);+ }+ }+ self.rebalanced.notify_one();+ }++ // NOTE: `ConsumerContext::commit_callback` is deliberately not implemented.+ // librdkafka only propagates a commit result to the application when a+ // conf-level `offset_commit_cb` is registered (rust-rdkafka never does) or+ // when the commit is synchronous (a replyq is attached); manual async+ // commits silently drop their result (`rd_kafka_cgrp_propagate_commit_result`).+}++#[cfg(test)]+mod tests {+ use rdkafka::config::ClientConfig;+ use rdkafka::consumer::Consumer;++ use super::*;++ #[test]+ fn partitions_sort_by_topic_then_partition_number() {+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition("overflow", 2);+ tpl.add_partition("events", 10);+ tpl.add_partition("events", 9);++ assert_eq!(+ partition_names(&topic_partitions(&tpl)),+ vec!["events:9", "events:10", "overflow:2"]+ );+ }++ struct CallbackCase {+ loop_running: bool,+ revoke_queued: bool,+ still_assigned: usize,+ }++ #[rstest::rstest]+ #[case::loop_running(CallbackCase { loop_running: true, revoke_queued: true, still_assigned: 2 })]+ #[case::no_loop(CallbackCase { loop_running: false, revoke_queued: false, still_assigned: 0 })]+ fn the_loop_applies_the_unassign_while_it_runs(#[case] case: CallbackCase) {+ let consumer: BaseConsumer<RebalanceContext> = ClientConfig::new()+ .set("bootstrap.servers", "127.0.0.1:1")+ .set("group.id", "test")+ .create_with_context(RebalanceContext::new())+ .expect("a client needs no broker to be built");+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition("events", 1);+ tpl.add_partition("events", 0);+ let context = consumer.context();+ if case.loop_running {+ context.on_started(None);+ }++ context.rebalance(+ &consumer,+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS,+ &mut tpl,+ );+ context.rebalance(+ &consumer,+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS,+ &mut tpl,+ );++ let partitions = vec![+ TopicPartition::new("events", 0),+ TopicPartition::new("events", 1),+ ];+ let mut expected = vec![Event::Assigned(partitions.clone())];+ if case.revoke_queued {+ expected.push(Event::Revoked {+ revoke: RevokeId(1),+ partitions,+ unassign: Unassign::new(tpl.clone(), false),+ });+ }+ assert_eq!(context.take_events(), expected);+ assert_eq!(consumer.assignment().unwrap().count(), case.still_assigned);+ }++ #[test]+ fn pausing_an_unassigned_partition_is_an_error() {+ let consumer: BaseConsumer = ClientConfig::new()+ .set("bootstrap.servers", "127.0.0.1:1")+ .set("group.id", "test")+ .create()+ .expect("a client needs no broker to be built");+ let mut tpl = TopicPartitionList::new();+ tpl.add_partition("events", 0);++ assert!(consumer.pause(&tpl).is_ok());+ assert!(partition_error(&tpl).is_err());+ }+}diff --git a/rust/ingestion-consumer/src/rebalance/mod.rs b/rust/ingestion-consumer/src/rebalance/mod.rsnew file mode 100644index 000000000000..6052edb5706d--- /dev/null+++ b/rust/ingestion-consumer/src/rebalance/mod.rs@@ -0,0 +1,6 @@+mod context;+mod state;++pub use context::RebalanceContext;+pub(crate) use context::{partition_error, partition_names};+pub(crate) use state::{Drain, Event, Unassign};diff --git a/rust/ingestion-consumer/src/rebalance/state.rs b/rust/ingestion-consumer/src/rebalance/state.rsnew file mode 100644index 000000000000..5b4f6eb5d849--- /dev/null+++ b/rust/ingestion-consumer/src/rebalance/state.rs@@ -0,0 +1,382 @@+use std::time::Duration;++use common_kafka_consumer::TopicPartition;+use rdkafka::consumer::{Consumer, ConsumerContext};+use rdkafka::TopicPartitionList;+use tokio::time::Instant;+use tracing::error;++use crate::batcher::RevokeId;++#[derive(Debug, PartialEq)]+pub(crate) enum Event {+ Assigned(Vec<TopicPartition>),+ Revoked {+ revoke: RevokeId,+ partitions: Vec<TopicPartition>,+ unassign: Unassign,+ },+ Draining {+ revoke: RevokeId,+ partitions: Vec<TopicPartition>,+ },+}++#[derive(Debug, PartialEq)]+pub(crate) struct Unassign {+ partitions: TopicPartitionList,+ cooperative: bool,+}++impl Unassign {+ pub(crate) fn new(partitions: TopicPartitionList, cooperative: bool) -> Self {+ Self {+ partitions,+ cooperative,+ }+ }++ pub(crate) fn apply<C: ConsumerContext>(self, consumer: &impl Consumer<C>) {+ let applied = if self.cooperative {+ consumer.incremental_unassign(&self.partitions)+ } else {+ consumer.unassign()+ };+ if let Err(err) = applied {+ error!(error = %err, "Could not unassign the revoked partitions");+ }+ }+}++pub(crate) struct Drain {+ revoke: RevokeId,+ pub(crate) partitions: Vec<TopicPartition>,+ pub(crate) unassign: Unassign,+ pub(crate) started: Instant,+ pub(crate) timeout: Duration,+}++enum Phase {+ Starting,+ Consuming { drain_timeout: Option<Duration> },+ Draining { drain: Drain },+ Stopped,+}++pub(crate) struct RebalanceState {+ phase: Phase,+ events: Vec<Event>,+ revokes: u64,+}++impl Default for RebalanceState {+ fn default() -> Self {+ Self {+ phase: Phase::Starting,+ events: Vec::new(),+ revokes: 0,+ }+ }+}++impl RebalanceState {+ pub(crate) fn on_started(&mut self, drain_timeout: Option<Duration>) {+ if matches!(self.phase, Phase::Starting) {+ self.phase = Phase::Consuming { drain_timeout };+ }+ }++ pub(crate) fn on_assigned(&mut self, partitions: Vec<TopicPartition>) {+ self.queue(Event::Assigned(partitions));+ }++ /// `Some` only before `on_started` or after `on_stopped`, when no loop+ /// runs to apply the unassign.+ pub(crate) fn on_revoked(+ &mut self,+ partitions: Vec<TopicPartition>,+ unassign: Unassign,+ lost: bool,+ now: Instant,+ ) -> Option<Unassign> {+ match self.phase {+ // A lost assignment belongs to another member, so its commit fails.+ Phase::Consuming {+ drain_timeout: Some(timeout),+ } if !lost => {+ let revoke = self.next_revoke();+ self.events.push(Event::Draining {+ revoke,+ partitions: partitions.clone(),+ });+ self.phase = Phase::Draining {+ drain: Drain {+ revoke,+ partitions,+ unassign,+ started: now,+ timeout,+ },+ };+ None+ }+ Phase::Consuming { .. } | Phase::Draining { .. } => {+ let revoke = self.next_revoke();+ self.events.push(Event::Revoked {+ revoke,+ partitions,+ unassign,+ });+ None+ }+ Phase::Starting | Phase::Stopped => Some(unassign),+ }+ }++ fn next_revoke(&mut self) -> RevokeId {+ self.revokes += 1;+ RevokeId(self.revokes)+ }++ pub(crate) fn take_events(&mut self) -> Vec<Event> {+ std::mem::take(&mut self.events)+ }++ pub(crate) fn drain_deadline(&self) -> Option<Instant> {+ match &self.phase {+ Phase::Draining { drain } => Some(drain.started + drain.timeout),+ _ => None,+ }+ }++ pub(crate) fn on_drained(&mut self, revoke: RevokeId) -> Option<Drain> {+ match &self.phase {+ Phase::Draining { drain } if drain.revoke == revoke => self.finish_drain(),+ _ => None,+ }+ }++ pub(crate) fn on_drain_timed_out(&mut self, now: Instant) -> Option<Drain> {+ match self.drain_deadline() {+ Some(deadline) if now >= deadline => self.finish_drain(),+ _ => None,+ }+ }++ pub(crate) fn on_stopped(&mut self) -> Option<Drain> {+ match std::mem::replace(&mut self.phase, Phase::Stopped) {+ Phase::Draining { drain } => Some(drain),+ _ => None,+ }+ }++ pub(crate) fn on_dropped(&mut self) -> Vec<Unassign> {+ let mut unassigns: Vec<Unassign> = self+ .on_stopped()+ .map(|drain| drain.unassign)+ .into_iter()+ .collect();+ unassigns.extend(+ self.take_events()+ .into_iter()+ .filter_map(|event| match event {+ Event::Revoked { unassign, .. } => Some(unassign),+ Event::Assigned(_) | Event::Draining { .. } => None,+ }),+ );+ unassigns+ }++ fn finish_drain(&mut self) -> Option<Drain> {+ match std::mem::replace(&mut self.phase, Phase::Stopped) {+ Phase::Draining { drain } => {+ self.phase = Phase::Consuming {+ drain_timeout: Some(drain.timeout),+ };+ Some(drain)+ }+ phase => {+ self.phase = phase;+ None+ }+ }+ }++ fn queue(&mut self, event: Event) {+ if !matches!(self.phase, Phase::Stopped) {+ self.events.push(event);+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ const TIMEOUT: Duration = Duration::from_secs(10);++ fn p(partition: i32) -> TopicPartition {+ TopicPartition::new("events", partition)+ }++ fn unassign() -> Unassign {+ Unassign::new(TopicPartitionList::new(), true)+ }++ fn consuming() -> RebalanceState {+ let mut state = RebalanceState::default();+ state.on_started(Some(TIMEOUT));+ state+ }++ #[test]+ fn a_revoke_while_consuming_waits_for_its_drain() {+ let mut state = consuming();+ let now = Instant::now();++ assert!(state+ .on_revoked(vec![p(0)], unassign(), false, now)+ .is_none());+ assert_eq!(+ state.take_events(),+ vec![Event::Draining {+ revoke: RevokeId(1),+ partitions: vec![p(0)],+ }]+ );+ assert_eq!(state.drain_deadline(), Some(now + TIMEOUT));+ assert!(+ state.on_drained(RevokeId(2)).is_none(),+ "another revoke's answer"+ );++ let drain = state.on_drained(RevokeId(1)).expect("the drain finishes");+ assert_eq!(drain.partitions, vec![p(0)]);+ assert_eq!(state.drain_deadline(), None);+ assert!(+ state+ .on_revoked(vec![p(1)], unassign(), false, now)+ .is_none(),+ "the next revoke drains too"+ );+ }++ fn without_drains() -> RebalanceState {+ let mut state = RebalanceState::default();+ state.on_started(None);+ state+ }++ fn draining() -> RebalanceState {+ let mut state = consuming();+ state.on_revoked(vec![p(9)], unassign(), false, Instant::now());+ state.take_events();+ state+ }++ fn stopped() -> RebalanceState {+ let mut state = consuming();+ state.on_stopped();+ state+ }++ struct LoopUnassignCase {+ state: fn() -> RebalanceState,+ lost: bool,+ revoke: RevokeId,+ }++ #[rstest::rstest]+ #[case::lost(LoopUnassignCase { state: consuming, lost: true, revoke: RevokeId(1) })]+ #[case::no_drain_timeout(LoopUnassignCase { state: without_drains, lost: false, revoke: RevokeId(1) })]+ #[case::during_a_drain(LoopUnassignCase { state: draining, lost: false, revoke: RevokeId(2) })]+ fn a_revoke_that_does_not_drain_goes_to_the_loop_with_its_unassign(+ #[case] case: LoopUnassignCase,+ ) {+ let mut state = (case.state)();++ assert!(state+ .on_revoked(vec![p(0)], unassign(), case.lost, Instant::now())+ .is_none());+ assert_eq!(+ state.take_events(),+ vec![Event::Revoked {+ revoke: case.revoke,+ partitions: vec![p(0)],+ unassign: unassign(),+ }]+ );+ }++ #[rstest::rstest]+ #[case::before_the_loop_starts(RebalanceState::default)]+ #[case::after_the_loop_stops(stopped)]+ fn without_a_loop_the_callback_applies_the_unassign(#[case] state: fn() -> RebalanceState) {+ let mut state = state();++ assert!(state+ .on_revoked(vec![p(0)], unassign(), false, Instant::now())+ .is_some());+ assert_eq!(state.take_events(), vec![]);+ assert_eq!(state.drain_deadline(), None);+ }++ #[test]+ fn a_timed_out_drain_answered_late_does_not_finish_the_next_drain_of_its_partitions() {+ let mut state = consuming();+ let now = Instant::now();+ state.on_revoked(vec![p(0)], unassign(), false, now);++ assert!(state.on_drain_timed_out(now + TIMEOUT / 2).is_none());+ assert!(state.on_drain_timed_out(now + TIMEOUT).is_some());++ state.on_revoked(vec![p(0)], unassign(), false, now + TIMEOUT);+ assert!(state.on_drained(RevokeId(1)).is_none());+ assert!(state.on_drained(RevokeId(2)).is_some());+ }++ #[test]+ fn stopping_hands_back_the_drain() {+ let mut state = draining();++ let drain = state.on_stopped().expect("the drain in progress");+ assert_eq!(drain.partitions, vec![p(9)]);+ }++ fn revoked_and_never_taken() -> RebalanceState {+ let mut state = without_drains();+ state.on_revoked(vec![p(0)], unassign(), false, Instant::now());+ state+ }++ fn draining_then_revoked_and_never_taken() -> RebalanceState {+ let mut state = draining();+ state.on_revoked(vec![p(0)], unassign(), false, Instant::now());+ state+ }++ struct DroppedCase {+ state: fn() -> RebalanceState,+ unassigns: usize,+ }++ #[rstest::rstest]+ #[case::nothing_pending(DroppedCase { state: consuming, unassigns: 0 })]+ #[case::a_revoke_the_loop_never_took(DroppedCase { state: revoked_and_never_taken, unassigns: 1 })]+ #[case::a_drain_in_progress(DroppedCase { state: draining, unassigns: 1 })]+ #[case::a_drain_and_a_revoke_the_loop_never_took(DroppedCase { state: draining_then_revoked_and_never_taken, unassigns: 2 })]+ #[case::after_the_loop_stopped(DroppedCase { state: stopped, unassigns: 0 })]+ fn a_dropped_loop_hands_back_every_unapplied_unassign(#[case] case: DroppedCase) {+ let mut state = (case.state)();++ assert_eq!(state.on_dropped().len(), case.unassigns);++ assert_eq!(state.take_events(), vec![]);+ assert!(+ state+ .on_revoked(vec![p(1)], unassign(), false, Instant::now())+ .is_some(),+ "stopped: the callback applies the next unassign itself"+ );+ }+}diff --git a/rust/ingestion-consumer/src/settlement/group.rs b/rust/ingestion-consumer/src/settlement/group.rsindex 4e8a0c7112a5..a69afc9f886a 100644--- a/rust/ingestion-consumer/src/settlement/group.rs+++ b/rust/ingestion-consumer/src/settlement/group.rs@@ -85,6 +85,16 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettleSome(Commit::new(handed_out, resume))}+ fn on_assigned(&self, assigned: &[TopicPartition]) {+ self.forget_partitions(assigned);+ }++ fn on_revoked(&self, revoked: &[TopicPartition]) {+ self.forget_partitions(revoked);+ }+}++impl<A: Accounting, D: Admission> GroupSettlement<A, D> {fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {self.accounting.forget_partitions(topic_partitions);self.admission.forget_partitions(topic_partitions);@@ -259,7 +269,7 @@ mod tests {fn a_revoke_reaches_both_the_accounting_and_the_admission() {let (settlement, accounting, admission) = settlement();- settlement.forget_partitions(&[p(0)]);+ settlement.on_revoked(&[p(0)]);assert_eq!(accounting.forgotten(), vec![p(0)]);assert_eq!(admission.forgotten(), vec![p(0)]);@@ -403,7 +413,7 @@ mod integration_tests {charge_offsets(&ledger, &p0, 0, &[11]);settlement.settle_run(&p0, 0, &[Offset(11)]).unwrap();- settlement.forget_partitions(&[p0]);+ settlement.on_revoked(&[p0]);assert!(settlement.take_all().is_none());}@@ -506,7 +516,7 @@ mod integration_tests {assert_eq!(charge(&settlement, &p0(), 10, 10), Some(Pause));settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();- settlement.forget_partitions(&[p0()]);+ settlement.on_revoked(&[p0()]);assert!(settlement.take_all().is_none());settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();@@ -515,4 +525,41 @@ mod integration_tests {"the stale run moves nothing");}++ #[test]+ fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {+ let ledger = Arc::new(TopicOffsetLedger::new());+ let settlement = GroupSettlement::new(+ Arc::clone(&ledger),+ Duration::from_secs(3600),+ BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),+ );+ let p0 = TopicPartition::new("events", 0);+ let one_event = [(+ Offset(10),+ Charge {+ events: 1,+ bytes: 0,+ },+ )];+ assert_eq!(+ settlement+ .charge(&p0, 0, &one_event)+ .map(|charged| charged.pause),+ Ok(Some(Pause))+ );+ settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();++ settlement.on_assigned(std::slice::from_ref(&p0));++ assert!(settlement.take_all().is_none());+ assert_eq!(ledger.generation(&p0), 1);+ assert_eq!(+ settlement+ .charge(&p0, 1, &one_event)+ .map(|charged| charged.pause),+ Ok(Some(Pause)),+ "the regained partition starts unpaused"+ );+ }}diff --git a/rust/ingestion-consumer/src/settlement/mod.rs b/rust/ingestion-consumer/src/settlement/mod.rsindex bb4fc2eea8a7..b74a1390e06e 100644--- a/rust/ingestion-consumer/src/settlement/mod.rs+++ b/rust/ingestion-consumer/src/settlement/mod.rs@@ -38,7 +38,9 @@ pub(crate) trait Settlement: Send + Sync {fn take_all(&self) -> Option<Commit>;- fn forget_partitions(&self, topic_partitions: &[TopicPartition]);+ fn on_assigned(&self, assigned: &[TopicPartition]);++ fn on_revoked(&self, revoked: &[TopicPartition]);}pub(crate) trait Accounting: Send + Sync {diff --git a/rust/ingestion-consumer/tests/batcher_integration_test.rs b/rust/ingestion-consumer/tests/batcher_integration_test.rsindex 5d99506a40d1..1a09d0ad21c0 100644--- a/rust/ingestion-consumer/tests/batcher_integration_test.rs+++ b/rust/ingestion-consumer/tests/batcher_integration_test.rs@@ -8,6 +8,7 @@ use common_kafka_consumer::Partition;use common::one_message_batcher;use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;+use ingestion_consumer::batcher::RevokeId;use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport};use ingestion_consumer::routing::RoutingStrategy;use ingestion_consumer::types::{Accumulator, SerializedKafkaMessage};@@ -59,7 +60,7 @@ async fn purging_a_just_submitted_batch_is_not_fatal() {accumulator.push(Partition(0), make_msg("a").into());batcher.submit(accumulator, &HashMap::new());// No await, so the task gets the submit and the purge before it runs.- batcher.revoker().purge_revoked(&[("test".to_string(), 0)]);+ batcher.revoke(RevokeId(1), &[("test".to_string(), 0)]);match tokio::time::timeout(Duration::from_millis(100), outputs.errors.recv()).await {Err(_) => {}diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rsindex 76b67c5285d4..608c8f064990 100644--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs@@ -2,7 +2,7 @@mod common;-use std::collections::HashSet;+use std::collections::{BTreeSet, HashMap, HashSet};use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};use std::sync::{Arc, Mutex, OnceLock};@@ -590,6 +590,33 @@ struct Harness {budget: Option<RequestBudget>,partition_budget: BudgetLimits,in_flight_limit: ChargeLimit,+ revoke_drain_timeout: Option<Duration>,+ rebalance_mode: RebalanceMode,+}++/// Production runs `group.protocol=consumer` (KIP-848), which also revokes+/// incrementally. The test broker, Redpanda v25.1.9, does not implement+/// ConsumerGroupHeartbeat (API key 68) and leaves the key out of its+/// ApiVersions response. So no case covers KIP-848; `Cooperative` takes the+/// same `incremental_unassign` path in the consumer.+#[derive(Clone, Copy, Debug, PartialEq)]+enum RebalanceMode {+ Eager,+ /// `cooperative-sticky`: a revoke takes only the partitions that move.+ /// The broker is `Stable` while the old owner holds them, and the new owner+ /// gets them in a second rebalance after the unassign.+ Cooperative,+}++impl RebalanceMode {+ fn configure(self, config: &mut ClientConfig) {+ match self {+ Self::Eager => {}+ Self::Cooperative => {+ config.set("partition.assignment.strategy", "cooperative-sticky");+ }+ }+ }}// Short session and heartbeat so group handovers finish quickly.@@ -598,7 +625,13 @@ fn make_kafka_consumer(group_id: &str,instance_id: Option<&str>,) -> StreamConsumer<RebalanceContext> {- make_kafka_consumer_with_context(topic, group_id, instance_id, RebalanceContext::new())+ make_kafka_consumer_with_context(+ topic,+ group_id,+ instance_id,+ RebalanceContext::new(),+ RebalanceMode::Eager,+ )}fn make_kafka_consumer_with_context(@@ -606,6 +639,7 @@ fn make_kafka_consumer_with_context(group_id: &str,instance_id: Option<&str>,context: RebalanceContext,+ rebalance_mode: RebalanceMode,) -> StreamConsumer<RebalanceContext> {let mut config = ClientConfig::new();config@@ -620,6 +654,7 @@ fn make_kafka_consumer_with_context(if let Some(id) = instance_id {config.set("group.instance.id", id);}+ rebalance_mode.configure(&mut config);let kafka_consumer: StreamConsumer<RebalanceContext> =config.create_with_context(context).expect("kafka consumer");kafka_consumer.subscribe(&[topic]).expect("subscribe");@@ -678,12 +713,22 @@ impl Harness {COMMIT_INTERVAL,BudgetLimits::unset(),ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,).await}/// One worker; every request carries a 500 ms soft budget.async fn start_budgeted(topic: &str, partitions: i32) -> Self {+ Self::start_budgeted_draining_revokes(topic, partitions, None).await+ }++ async fn start_budgeted_draining_revokes(+ topic: &str,+ partitions: i32,+ revoke_drain_timeout: Option<Duration>,+ ) -> Self {Self::start_inner(topic,partitions,@@ -706,6 +751,8 @@ impl Harness {COMMIT_INTERVAL,BudgetLimits::unset(),ChargeLimit::unset(),+ revoke_drain_timeout,+ RebalanceMode::Eager,).await}@@ -731,6 +778,8 @@ impl Harness {COMMIT_INTERVAL,BudgetLimits::unset(),ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,).await}@@ -757,6 +806,34 @@ impl Harness {COMMIT_INTERVAL,BudgetLimits::unset(),ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,+ )+ .await+ }++ async fn start_draining_revokes(+ topic: &str,+ partitions: i32,+ worker_count: usize,+ revoke_drain_timeout: Duration,+ rebalance_mode: RebalanceMode,+ ) -> Self {+ Self::start_inner(+ topic,+ partitions,+ worker_count,+ Duration::from_secs(60),+ fast_registry_config(),+ 0,+ ComponentOptions::new(),+ common::PRODUCTION_PACK_TARGETS,+ None,+ COMMIT_INTERVAL,+ BudgetLimits::unset(),+ ChargeLimit::unset(),+ Some(revoke_drain_timeout),+ rebalance_mode,).await}@@ -775,6 +852,8 @@ impl Harness {COMMIT_INTERVAL,BudgetLimits::unset(),ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,).await}@@ -799,6 +878,8 @@ impl Harness {commit_interval,BudgetLimits::unset(),ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,).await}@@ -842,6 +923,8 @@ impl Harness {COMMIT_INTERVAL,partition_budget,in_flight_limit,+ None,+ RebalanceMode::Eager,).await}@@ -867,6 +950,8 @@ impl Harness {commit_interval,partition_budget,ChargeLimit::unset(),+ None,+ RebalanceMode::Eager,).await}@@ -885,6 +970,8 @@ impl Harness {commit_interval: Duration,partition_budget: BudgetLimits,in_flight_limit: ChargeLimit,+ revoke_drain_timeout: Option<Duration>,+ rebalance_mode: RebalanceMode,) -> Self {create_topic(topic, partitions).await;@@ -914,8 +1001,13 @@ impl Harness {let group_id = format!("e2e-{}", Uuid::new_v4());let ledger = Arc::new(TopicOffsetLedger::new());let debug_recorder = DebugRecorder::new(10_000, Duration::from_secs(3600));- let kafka_consumer =- make_kafka_consumer_with_context(topic, &group_id, None, RebalanceContext::new());+ let kafka_consumer = make_kafka_consumer_with_context(+ topic,+ &group_id,+ None,+ RebalanceContext::new(),+ rebalance_mode,+ );let (batcher, outputs) = common::batcher(pool_source,@@ -941,6 +1033,7 @@ impl Harness {ledger: Arc::clone(&ledger),partition_budget,in_flight_limit,+ revoke_drain_timeout,},handle,);@@ -974,6 +1067,8 @@ impl Harness {budget,partition_budget,in_flight_limit,+ revoke_drain_timeout,+ rebalance_mode,}}@@ -989,6 +1084,19 @@ impl Harness {.collect()}+ fn dispatched_assignment_epochs(&self) -> Vec<u64> {+ self.debug_recorder+ .backlog()+ .into_iter()+ .filter_map(|event| match event.kind {+ DebugEventKind::BatchDispatched {+ assignment_epoch, ..+ } => Some(assignment_epoch),+ _ => None,+ })+ .collect()+ }+/// Abort the consumer task: no shutdown, no drain, no final commit.fn crash_consumer(&mut self) {if let Some(task) = self.task.take() {@@ -1017,6 +1125,7 @@ impl Harness {&self.group_id,None,RebalanceContext::new(),+ self.rebalance_mode,);let (batcher, outputs) = common::batcher(pool_source,@@ -1042,6 +1151,7 @@ impl Harness {ledger: Arc::clone(&self.ledger),partition_budget: self.partition_budget,in_flight_limit: self.in_flight_limit,+ revoke_drain_timeout: self.revoke_drain_timeout,},handle,);@@ -1113,7 +1223,13 @@ impl Harness {self.budget,);let consumer = IngestionConsumer::from_parts(- make_kafka_consumer(&self.topic, &self.group_id, None),+ make_kafka_consumer_with_context(+ &self.topic,+ &self.group_id,+ None,+ RebalanceContext::new(),+ self.rebalance_mode,+ ),batcher,outputs,transport,@@ -1128,6 +1244,7 @@ impl Harness {ledger: Arc::new(TopicOffsetLedger::new()),partition_budget: self.partition_budget,in_flight_limit: self.in_flight_limit,+ revoke_drain_timeout: self.revoke_drain_timeout,},handle,);@@ -1187,6 +1304,97 @@ fn broker_committed_offset(group_id: &str, topic: &str, partition: i32) -> Optio}}+#[derive(Debug)]+struct GroupState {+ state: String,+ owned: Vec<usize>,+}++impl GroupState {+ fn handed_over(&self) -> bool {+ self.state == "Stable" && self.owned == [1, 1]+ }++ fn holds_a_revoke(&self, mode: RebalanceMode) -> bool {+ match mode {+ RebalanceMode::Eager => self.state == "PreparingRebalance" && self.owned.len() == 2,+ RebalanceMode::Cooperative => self.state == "Stable" && self.owned == [0, 1],+ }+ }+}++fn group_state(group_id: &str) -> Option<GroupState> {+ let probe: BaseConsumer = ClientConfig::new()+ .set("bootstrap.servers", KAFKA_BROKERS)+ .set("group.id", group_id)+ .create()+ .expect("group probe");+ let groups = probe+ .client()+ .fetch_group_list(Some(group_id), Duration::from_secs(5))+ .ok()?;+ let group = groups+ .groups()+ .iter()+ .find(|group| group.name() == group_id)?;+ let mut owned: Vec<usize> = group+ .members()+ .iter()+ .map(|member| assigned_partition_count(member.assignment().unwrap_or_default()))+ .collect();+ owned.sort_unstable();+ Some(GroupState {+ state: group.state().to_string(),+ owned,+ })+}++/// Counts the partitions in a classic-protocol `ConsumerProtocolAssignment`:+/// an i16 version, then an i32 count of (i16-length topic, i32 count of i32 partitions).+fn assigned_partition_count(assignment: &[u8]) -> usize {+ fn read<const N: usize>(bytes: &[u8], at: &mut usize) -> Option<[u8; N]> {+ let read = bytes.get(*at..*at + N)?.try_into().ok()?;+ *at += N;+ Some(read)+ }+ fn count(assignment: &[u8]) -> Option<usize> {+ let mut at = 2;+ let topics = i32::from_be_bytes(read(assignment, &mut at)?);+ let mut partitions = 0;+ for _ in 0..topics {+ at += i16::from_be_bytes(read(assignment, &mut at)?) as usize;+ let in_topic = i32::from_be_bytes(read(assignment, &mut at)?) as usize;+ at += 4 * in_topic;+ partitions += in_topic;+ }+ Some(partitions)+ }+ if assignment.is_empty() {+ return 0;+ }+ count(assignment).expect("a well-formed consumer protocol assignment")+}++async fn wait_for_group(+ group_id: &str,+ timeout: Duration,+ what: &str,+ cond: impl Fn(&GroupState) -> bool,+) {+ let deadline = tokio::time::Instant::now() + timeout;+ loop {+ let observed = group_state(group_id);+ if observed.as_ref().is_some_and(&cond) {+ return;+ }+ assert!(+ tokio::time::Instant::now() < deadline,+ "timed out waiting for the group to {what}, last seen {observed:?}"+ );+ tokio::time::sleep(Duration::from_millis(250)).await;+ }+}+async fn wait_until(timeout: Duration, msg: &str, mut cond: impl FnMut() -> bool) {let deadline = tokio::time::Instant::now() + timeout;while !cond() {@@ -1459,6 +1667,100 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {harness.stop().await;}+#[tokio::test]+async fn a_partial_ack_during_a_drain_commits_only_what_was_processed() {+ let topic = format!("e2e-partial-drain-{}", Uuid::new_v4());+ let harness =+ Harness::start_budgeted_draining_revokes(&topic, 1, Some(Duration::from_secs(20))).await;+ // The partition holds a, b, a, b, and the b messages come back unfinished.+ *harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-b".to_string(), 1));+ let records = alternating_visitors(0, "tok:203.0.113.7", 0);++ // A trickle keeps the consumer polling, so it takes part in the rebalance.+ let producer = make_producer();+ producer+ .client()+ .fetch_metadata(Some(&topic), Duration::from_secs(5))+ .expect("trickle producer metadata");++ let gate = harness.workers[0].block().await;+ produce_keyed_in_one_transaction(&topic, "tok", &records).await;+ let trickling = Arc::new(AtomicBool::new(true));+ let trickle = tokio::spawn({+ let trickling = Arc::clone(&trickling);+ let topic = topic.clone();+ async move {+ let mut seq = 1000usize;+ while trickling.load(Ordering::SeqCst) {+ produce(&producer, &topic, 0, "tok", "trickle", seq).await;+ seq += 1;+ tokio::time::sleep(Duration::from_millis(20)).await;+ }+ }+ });+ wait_until(+ Duration::from_secs(15),+ "a request to reach the worker",+ || harness.workers[0].arrived_count() > 0,+ )+ .await;++ let second = harness.join_second_consumer().await;+ // The drain waits on the gated request.+ tokio::time::sleep(Duration::from_secs(2)).await;+ trickling.store(false, Ordering::SeqCst);+ trickle.await.expect("trickle task");+ drop(gate);++ wait_until(+ Duration::from_secs(30),+ "every message to be delivered",+ || {+ let delivered: HashSet<(String, usize)> = harness+ .delivery_log+ .lock()+ .unwrap()+ .iter()+ .cloned()+ .collect();+ records+ .iter()+ .all(|(_, _, visitor, seq)| delivered.contains(&(visitor.to_string(), *seq)))+ },+ )+ .await;+ // Time for a replay by the new owner to arrive.+ tokio::time::sleep(Duration::from_secs(2)).await;++ assert!(+ harness.workers[0]+ .leave_unfinished+ .lock()+ .unwrap()+ .is_none(),+ "the gated request was acked as partial"+ );+ assert_eq!(+ *harness.workers[0].replayed.lock().unwrap(),+ Vec::<(String, usize)>::new(),+ "the old owner resends nothing of a draining partition"+ );+ let log = harness.delivery_log.lock().unwrap().clone();+ let mut seen = HashSet::new();+ let processed_twice: BTreeSet<(String, usize)> = log+ .iter()+ .filter(|entry| entry.0 != "trickle" && !seen.insert(*entry))+ .cloned()+ .collect();+ assert_eq!(+ processed_twice,+ BTreeSet::from([("visitor-a".to_string(), 2)])+ );++ second.stop().await;+ harness.stop().await;+}+#[tokio::test]async fn a_message_that_never_fits_the_budget_lands_without_one() {let topic = format!("e2e-partial-escalate-{}", Uuid::new_v4());@@ -3370,6 +3672,321 @@ async fn second_consumer_joining_the_group_preserves_all_messages() {harness.stop().await;}+#[rstest]+#[case::eager(RebalanceMode::Eager)]+#[case::cooperative(RebalanceMode::Cooperative)]+#[tokio::test]+async fn a_drain_commits_the_in_flight_messages_before_the_new_owner_reads(+ #[case] rebalance_mode: RebalanceMode,+) {+ let topic = format!("e2e-revoke-drain-{}", Uuid::new_v4());+ let harness =+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)+ .await;+ let producer = make_producer();++ let mut guards = Vec::new();+ for w in &harness.workers {+ guards.push(w.block().await);+ }+ for seq in 0..5usize {+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;+ }+ wait_until(Duration::from_secs(10), "work to reach a worker", || {+ harness.workers.iter().any(|w| w.arrived_count() > 0)+ })+ .await;++ let second = harness.join_second_consumer().await;+ // The drain waits on the gated requests.+ tokio::time::sleep(Duration::from_secs(2)).await;+ guards.clear();+ for seq in 5..10usize {+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;+ }++ let expected: HashSet<(String, usize)> = ["user-1", "user-2"]+ .iter()+ .flat_map(|user| (0..10usize).map(|seq| (user.to_string(), seq)))+ .collect();+ wait_until(+ Duration::from_secs(30),+ "every message to be delivered",+ || {+ let delivered: HashSet<(String, usize)> = harness+ .delivery_log+ .lock()+ .unwrap()+ .iter()+ .cloned()+ .collect();+ expected.is_subset(&delivered)+ },+ )+ .await;+ // Time for a replay by the new owner to arrive.+ tokio::time::sleep(Duration::from_secs(2)).await;++ let log = harness.delivery_log.lock().unwrap().clone();+ let mut seen = HashSet::new();+ let duplicates: Vec<&(String, usize)> =+ log.iter().filter(|entry| !seen.insert(*entry)).collect();+ assert!(+ duplicates.is_empty(),+ "the new owner processed again what the old owner finished: {duplicates:?}"+ );++ second.stop().await;+ harness.stop().await;+}++fn processed_counts(harness: &Harness) -> HashMap<(String, usize), usize> {+ let mut counts = HashMap::new();+ for entry in harness.delivery_log.lock().unwrap().iter() {+ *counts.entry(entry.clone()).or_insert(0) += 1;+ }+ counts+}++#[rstest]+#[case::eager(RebalanceMode::Eager)]+#[case::cooperative(RebalanceMode::Cooperative)]+#[tokio::test]+async fn a_drain_holds_every_partition_and_dispatches_nothing_until_its_release(+ #[case] rebalance_mode: RebalanceMode,+) {+ let topic = format!("e2e-drain-hold-{}", Uuid::new_v4());+ let harness =+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)+ .await;+ let producer = make_producer();++ let mut guards = Vec::new();+ for w in &harness.workers {+ guards.push(w.block().await);+ }+ for seq in 0..5usize {+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;+ }+ wait_until(+ Duration::from_secs(10),+ "every produced message to dispatch",+ || harness.poll_sizes().iter().sum::<usize>() == 10,+ )+ .await;++ let second = harness.join_second_consumer().await;+ wait_for_group(+ &harness.group_id,+ Duration::from_secs(10),+ "hold a revoke",+ |group| group.holds_a_revoke(rebalance_mode),+ )+ .await;+ // The revoke arrives on the next heartbeat; the gated requests hold the drain.+ tokio::time::sleep(Duration::from_secs(2)).await;+ let dispatched = harness.poll_sizes().len();+ for seq in 5..15usize {+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;+ }+ tokio::time::sleep(Duration::from_secs(2)).await;+ assert_eq!(+ harness.poll_sizes().len(),+ dispatched,+ "librdkafka pauses the whole assignment until the unassign, so a drain dispatches no poll"+ );+ let group = group_state(&harness.group_id);+ assert!(+ group+ .as_ref()+ .is_some_and(|group| group.holds_a_revoke(rebalance_mode)),+ "the drain still holds the revoke: {group:?}"+ );++ guards.clear();+ let expected: HashSet<(String, usize)> = ["user-1", "user-2"]+ .iter()+ .flat_map(|user| (0..15usize).map(|seq| (user.to_string(), seq)))+ .collect();+ wait_until(+ Duration::from_secs(30),+ "every message to be delivered",+ || {+ let delivered: HashSet<(String, usize)> = harness+ .delivery_log+ .lock()+ .unwrap()+ .iter()+ .cloned()+ .collect();+ expected.is_subset(&delivered)+ },+ )+ .await;+ wait_for_group(+ &harness.group_id,+ Duration::from_secs(10),+ "hand the partition over",+ GroupState::handed_over,+ )+ .await;+ // Time for a replay by either owner to arrive.+ tokio::time::sleep(Duration::from_secs(2)).await;++ let duplicates: Vec<((String, usize), usize)> = processed_counts(&harness)+ .into_iter()+ .filter(|(_, count)| *count > 1)+ .collect();+ assert!(+ duplicates.is_empty(),+ "processed more than once: {duplicates:?}"+ );++ second.stop().await;+ harness.stop().await;+}++#[rstest]+#[case::eager(RebalanceMode::Eager)]+#[case::cooperative(RebalanceMode::Cooperative)]+#[tokio::test]+async fn a_drain_that_times_out_hands_the_partitions_over_uncommitted(+ #[case] rebalance_mode: RebalanceMode,+) {+ let topic = format!("e2e-drain-timeout-{}", Uuid::new_v4());+ let harness =+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(3), rebalance_mode).await;+ let producer = make_producer();+ let partitions = [+ TopicPartition::new(topic.clone(), 0),+ TopicPartition::new(topic.clone(), 1),+ ];+ let users = [("user-1", 0), ("user-2", 1)];++ let mut guards = Vec::new();+ for w in &harness.workers {+ guards.push(w.block().await);+ }+ for seq in 0..5usize {+ for (user, partition) in users {+ produce(&producer, &topic, partition, "tok", user, seq).await;+ }+ }+ wait_until(+ Duration::from_secs(10),+ "every produced message to dispatch",+ || harness.poll_sizes().iter().sum::<usize>() == 10,+ )+ .await;+ let generations_before: Vec<u64> = partitions+ .iter()+ .map(|partition| harness.ledger.generation(partition))+ .collect();+ let bumps = |harness: &Harness| -> Vec<u64> {+ let mut bumps: Vec<u64> = partitions+ .iter()+ .zip(&generations_before)+ .map(|(partition, before)| harness.ledger.generation(partition) - before)+ .collect();+ bumps.sort_unstable();+ bumps+ };++ let joined = tokio::time::Instant::now();+ let second = harness.join_second_consumer().await;+ // The gated requests outlive the drain timeout.+ wait_for_group(+ &harness.group_id,+ Duration::from_secs(20),+ "hand the partition over",+ GroupState::handed_over,+ )+ .await;+ assert!(+ joined.elapsed() >= Duration::from_secs(3),+ "the drain held the revoke for its timeout"+ );+ assert_eq!(harness.committed_offset(0), None);+ assert_eq!(harness.committed_offset(1), None);+ // The release bumps the generation of each revoked partition, and an+ // assignment bumps the generation of each partition it returns.+ let expected_bumps = match rebalance_mode {+ RebalanceMode::Eager => vec![1, 2],+ RebalanceMode::Cooperative => vec![0, 1],+ };+ wait_until(+ Duration::from_secs(10),+ "the first consumer to apply its new assignment",+ || bumps(&harness) == expected_bumps,+ )+ .await;+ let moved = partitions+ .iter()+ .zip(&generations_before)+ .find(|(partition, before)| harness.ledger.generation(partition) == *before + 1)+ .map(|(partition, _)| partition.partition)+ .expect("one partition left the first consumer");++ guards.clear();+ let read_again = |partition: i32| partition == moved || rebalance_mode == RebalanceMode::Eager;+ let processed_as_expected = |harness: &Harness| {+ let counts = processed_counts(harness);+ users.iter().all(|(user, partition)| {+ let expected = if read_again(*partition) { 2 } else { 1 };+ (0..5usize).all(|seq| counts.get(&(user.to_string(), seq)) == Some(&expected))+ })+ };+ wait_until(+ Duration::from_secs(30),+ "the released requests and the new owners to process every message",+ || processed_as_expected(&harness),+ )+ .await;+ wait_until(+ Duration::from_secs(10),+ "each owner to commit its partition",+ || harness.committed_offset(0) == Some(5) && harness.committed_offset(1) == Some(5),+ )+ .await;+ // Time for a replay by either owner to arrive.+ tokio::time::sleep(Duration::from_secs(2)).await;++ assert!(+ processed_as_expected(&harness),+ "the old owner's released requests process every message once, and each new assignment processes its partition once more: {:?}",+ processed_counts(&harness)+ );+ assert_eq!(+ bumps(&harness),+ expected_bumps,+ "no later rebalance touched the first consumer's partitions"+ );+ let commits_of_moved: Vec<i64> = harness+ .debug_recorder+ .backlog()+ .into_iter()+ .filter_map(|event| match event.kind {+ DebugEventKind::CommitSubmitted { offsets } => Some(offsets),+ _ => None,+ })+ .flatten()+ .filter(|offset| offset.partition == moved)+ .map(|offset| offset.offset)+ .collect();+ assert_eq!(+ commits_of_moved,+ Vec::<i64>::new(),+ "the old owner's late completions commit nothing for the partition it handed over"+ );++ second.stop().await;+ harness.stop().await;+}+#[tokio::test]async fn partition_lost_and_regained_keeps_the_consumer_alive() {let topic = format!("e2e-regain-{}", Uuid::new_v4());@@ -3471,6 +4088,11 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {},).await;+ let epochs = harness.dispatched_assignment_epochs();+ assert!(+ epochs.last() > epochs.first(),+ "polls after the regain go out under a later assignment epoch: {epochs:?}"+ );harness.stop().await;}@@ -3516,6 +4138,7 @@ async fn fenced_static_member_exits_on_fatal_error() {ledger: Arc::new(TopicOffsetLedger::new()),partition_budget: BudgetLimits::unset(),in_flight_limit: ChargeLimit::unset(),+ revoke_drain_timeout: None,},handle,);diff --git a/rust/ingestion-consumer/tests/grpc_transport_test.rs b/rust/ingestion-consumer/tests/grpc_transport_test.rsindex ef9f4092a13f..79f980f56cf8 100644--- a/rust/ingestion-consumer/tests/grpc_transport_test.rs+++ b/rust/ingestion-consumer/tests/grpc_transport_test.rs@@ -9,6 +9,7 @@ use std::time::Duration;use common::one_message_batcher;use common_kafka_consumer::Partition;use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;+use ingestion_consumer::batcher::BatcherCompletion;use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport, SendOptions};use ingestion_consumer::routing::RoutingStrategy;use ingestion_consumer::transport::{SendOutcome, TransportError};@@ -1396,6 +1397,9 @@ async fn a_busy_send_is_retried_until_it_completes() {.await.expect("successful retry completes").expect("completion channel stays open");+ let BatcherCompletion::Group(completion) = completion else {+ panic!("expected a group completion");+ };assert_eq!(completion.accepted, 1);tokio::time::sleep(Duration::from_millis(550)).await;assert!(@@ -1440,6 +1444,9 @@ async fn the_watchdog_allows_in_flight_success_after_the_deadline() {.await.expect("late successful send completes").expect("completion channel stays open");+ let BatcherCompletion::Group(completion) = completion else {+ panic!("expected a group completion");+ };assert_eq!(completion.accepted, 1);tokio::time::sleep(Duration::from_millis(150)).await;