Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): commit revoked partitions before releasing them (#126503)

Paweł Ledwoń250c0af11fc2
diff --git a/rust/ingestion-consumer/src/batcher/drains.rs b/rust/ingestion-consumer/src/batcher/drains.rs
new file mode 100644
index 000000000000..8c03d90a8572
--- /dev/null
+++ b/rust/ingestion-consumer/src/batcher/drains.rs
@@ -0,0 +1,37 @@
+use std::collections::HashSet;
+
+use super::in_flight::RequestId;
+use super::RevokeId;
+
+#[derive(Default)]
+pub struct PartitionDrains {
+ pending: Vec<PartitionDrain>,
+}
+
+struct PartitionDrain {
+ revoke: RevokeId,
+ carrying: HashSet<RequestId>,
+}
+
+impl PartitionDrains {
+ pub fn on_revoked(&mut self, revoke: RevokeId, carrying: impl IntoIterator<Item = RequestId>) {
+ self.pending.push(PartitionDrain {
+ revoke,
+ carrying: carrying.into_iter().collect(),
+ });
+ }
+
+ pub fn on_request_settled(&mut self, request: RequestId) {
+ for drain in &mut self.pending {
+ drain.carrying.remove(&request);
+ }
+ }
+
+ pub fn take_drained(&mut self) -> Vec<RevokeId> {
+ let (drained, pending): (Vec<_>, Vec<_>) = std::mem::take(&mut self.pending)
+ .into_iter()
+ .partition(|drain| drain.carrying.is_empty());
+ self.pending = pending;
+ drained.into_iter().map(|drain| drain.revoke).collect()
+ }
+}
diff --git a/rust/ingestion-consumer/src/batcher/driver.rs b/rust/ingestion-consumer/src/batcher/driver.rs
index 88b775bab316..f059a0903686 100644
--- a/rust/ingestion-consumer/src/batcher/driver.rs
+++ b/rust/ingestion-consumer/src/batcher/driver.rs
@@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
-use common_kafka_consumer::{AssignmentEpoch, GroupCompletion, Partition};
+use common_kafka_consumer::{AssignmentEpoch, Partition};
use futures::future::{BoxFuture, FutureExt};
use futures::stream::{FuturesUnordered, StreamExt};
use metrics::{counter, histogram};
@@ -20,7 +20,7 @@ use super::state_machine::{
BatcherSnapshot, BatcherStateMachine, Effects, FailureCause, SendRequest,
};
use super::worker_pool::WorkerPoolSource;
-use super::{make_batch_id, BatcherOutputs};
+use super::{make_batch_id, BatcherCompletion, BatcherOutputs, RevokeId};
use crate::debug_recorder::{partition_offsets, record_if, DebugEventKind, DebugRecorder};
use crate::grpc_transport::{GrpcTransport, SendOptions};
use crate::order_sentinel::{KeyOrderSentinel, SendKind, SentinelBatch};
@@ -44,7 +44,10 @@ enum Input {
assignment_epoch: u64,
runs: Vec<KeyRun>,
},
- PartitionsRevoked(Vec<(String, i32)>),
+ PartitionsRevoked {
+ revoke: RevokeId,
+ partitions: Vec<(String, i32)>,
+ },
Shutdown,
/// Answered between two actions, so the snapshot is consistent.
Snapshot(oneshot::Sender<BatcherSnapshot>),
@@ -60,8 +63,6 @@ pub(super) struct Load {
pub adaptation: Adaptation,
}
-pub(super) struct RevokeSender(mpsc::UnboundedSender<Input>);
-
#[derive(Clone)]
pub(super) struct SnapshotSender(mpsc::UnboundedSender<Input>);
@@ -140,7 +141,7 @@ struct BatcherTask<S, W> {
workers: W,
sender: Arc<S>,
key_sentinel: Arc<KeyOrderSentinel>,
- completions: mpsc::UnboundedSender<GroupCompletion>,
+ completions: mpsc::UnboundedSender<BatcherCompletion>,
errors: mpsc::UnboundedSender<String>,
load: Arc<Load>,
recorder: Option<Arc<DebugRecorder>>,
@@ -207,10 +208,6 @@ impl StateMachineDriver {
Arc::clone(&self.load)
}
- pub(super) fn revoke_sender(&self) -> RevokeSender {
- RevokeSender(self.inputs.clone())
- }
-
pub(super) fn snapshot_sender(&self) -> SnapshotSender {
SnapshotSender(self.inputs.clone())
}
@@ -243,6 +240,13 @@ impl StateMachineDriver {
self.send(Input::Shutdown);
}
+ pub(super) fn revoke(&self, revoke: RevokeId, partitions: &[(String, i32)]) {
+ self.send(Input::PartitionsRevoked {
+ revoke,
+ partitions: partitions.to_vec(),
+ });
+ }
+
fn send(&self, input: Input) {
if self.inputs.send(input).is_err() {
error!("Batcher task is gone");
@@ -256,15 +260,6 @@ impl Drop for StateMachineDriver {
}
}
-impl RevokeSender {
- /// Called from the rebalance callback inside the consumer loop's Kafka
- /// poll, so it queues the revoke rather than waiting for it. The revoke
- /// lands between the polls submitted before and after the rebalance.
- pub(super) fn purge_revoked(&self, partitions: &[(String, i32)]) {
- let _ = self.0.send(Input::PartitionsRevoked(partitions.to_vec()));
- }
-}
-
impl<S: RequestSender, W: Workers> BatcherTask<S, W> {
async fn run(mut self, mut state: BatcherStateMachine) {
loop {
@@ -306,11 +301,11 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {
assigned = true;
state.on_groups(now, &self.workers.candidates(), assignment_epoch, runs)
}
- Event::Input(Input::PartitionsRevoked(partitions)) => {
+ Event::Input(Input::PartitionsRevoked { revoke, partitions }) => {
// Cleared in order with the sends, so no revoked message is
// noted as sent after the revoke.
self.key_sentinel.clear();
- state.on_partitions_revoked(now, &partitions)
+ state.on_partitions_revoked(now, revoke, &partitions)
}
Event::Input(Input::Shutdown) => state.on_shutdown(now, &self.workers.candidates()),
Event::Input(Input::Snapshot(_)) => unreachable!("answered above"),
@@ -368,6 +363,7 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {
let Effects {
sends,
completions,
+ drained,
key_acks,
evicted_keys,
worker_outcomes,
@@ -408,7 +404,10 @@ impl<S: RequestSender, W: Workers> BatcherTask<S, W> {
counter!("ingestion_consumer_group_completions_total").increment(1);
counter!("ingestion_consumer_group_completion_accepted_messages_total")
.increment(completion.accepted as u64);
- let _ = self.completions.send(completion);
+ let _ = self.completions.send(BatcherCompletion::Group(completion));
+ }
+ for revoke in drained {
+ let _ = self.completions.send(BatcherCompletion::Drained(revoke));
}
if let Some(reason) = fatal {
error!(error = %reason, "Batcher failure");
@@ -687,14 +686,38 @@ mod tests {
}
fn completed_offsets(&mut self) -> Vec<i64> {
- let mut completed = Vec::new();
+ self.received()
+ .into_iter()
+ .filter_map(|received| match received {
+ Received::Completed(offset) => Some(offset),
+ Received::Drained(_) => None,
+ })
+ .collect()
+ }
+
+ fn received(&mut self) -> Vec<Received> {
+ let mut received = Vec::new();
while let Ok(completion) = self.outputs.completions.try_recv() {
- completed.extend(completion.offsets.iter().map(|offset| offset.0));
+ match completion {
+ BatcherCompletion::Group(group) => received.extend(
+ group
+ .offsets
+ .iter()
+ .map(|offset| Received::Completed(offset.0)),
+ ),
+ BatcherCompletion::Drained(revoke) => received.push(Received::Drained(revoke)),
+ }
}
- completed
+ received
}
}
+ #[derive(Debug, PartialEq)]
+ enum Received {
+ Completed(i64),
+ Drained(RevokeId),
+ }
+
impl FakeWorkers {
fn set_candidates(&self, workers: &[&str]) {
self.0.lock().unwrap().candidates =
@@ -869,9 +892,7 @@ mod tests {
let send = h.take_sent().pop().unwrap();
send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();
- h.driver
- .revoke_sender()
- .purge_revoked(&[("events".to_string(), 0)]);
+ h.driver.revoke(RevokeId(1), &[("events".to_string(), 0)]);
settle().await;
assert!(
@@ -889,13 +910,54 @@ mod tests {
let sentinel = h.driver.key_order_sentinel();
assert_eq!(sentinel.key_count(), 1);
- h.driver
- .revoke_sender()
- .purge_revoked(&[("events".to_string(), 9)]);
+ h.driver.revoke(RevokeId(1), &[("events".to_string(), 9)]);
settle().await;
assert_eq!(sentinel.key_count(), 0);
}
+ #[tokio::test(start_paused = true)]
+ async fn a_drain_signals_once_no_request_in_flight_carries_its_partitions() {
+ let mut h = Harness::new(&["w"], 4);
+ h.submit(vec![run("a", &[1])]);
+ settle().await;
+ let send = h.take_sent().pop().unwrap();
+ let (idle, busy) = (RevokeId(1), RevokeId(2));
+
+ h.driver.revoke(idle, &[("events".to_string(), 9)]);
+ h.driver.revoke(busy, &[("events".to_string(), 0)]);
+ settle().await;
+ assert_eq!(h.received(), vec![Received::Drained(idle)]);
+
+ send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();
+ settle().await;
+ assert_eq!(
+ h.received(),
+ vec![Received::Completed(1), Received::Drained(busy)]
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn a_drain_waits_for_a_poll_submitted_just_before_it() {
+ let mut h = Harness::new(&["w"], 4);
+ let revoke = RevokeId(1);
+
+ h.submit(vec![run("a", &[1])]);
+ h.driver.revoke(revoke, &[("events".to_string(), 0)]);
+ settle().await;
+ let send = h
+ .take_sent()
+ .pop()
+ .expect("the poll went out before the drain");
+ assert_eq!(h.received(), vec![]);
+
+ send.reply.send(Ok(SendOutcome::Accepted(1))).unwrap();
+ settle().await;
+ assert_eq!(
+ h.received(),
+ vec![Received::Completed(1), Received::Drained(revoke)]
+ );
+ }
+
#[tokio::test(start_paused = true)]
async fn the_load_tracks_pending_in_flight_and_busy_workers() {
let h = Harness::new(&[], 2);
diff --git a/rust/ingestion-consumer/src/batcher/in_flight.rs b/rust/ingestion-consumer/src/batcher/in_flight.rs
index f8836955ce37..3e91e8d7f91e 100644
--- a/rust/ingestion-consumer/src/batcher/in_flight.rs
+++ b/rust/ingestion-consumer/src/batcher/in_flight.rs
@@ -127,6 +127,25 @@ impl InFlightRequests {
pub fn take(&mut self, id: RequestId) -> Option<InFlightRequest> {
self.requests.remove(&id)
}
+
+ /// Matches partition numbers only, because a sent message carries no topic.
+ pub fn carrying<'a>(
+ &'a self,
+ partitions: &'a [(String, i32)],
+ ) -> impl Iterator<Item = RequestId> + 'a {
+ self.requests
+ .iter()
+ .filter(|(_, request)| {
+ request.runs.iter().any(|run| {
+ run.messages.iter().any(|message| {
+ partitions
+ .iter()
+ .any(|(_, partition)| *partition == message.partition)
+ })
+ })
+ })
+ .map(|(id, _)| *id)
+ }
}
impl InFlightRequest {
diff --git a/rust/ingestion-consumer/src/batcher/mod.rs b/rust/ingestion-consumer/src/batcher/mod.rs
index ab9660a644de..e7657de9f5bb 100644
--- a/rust/ingestion-consumer/src/batcher/mod.rs
+++ b/rust/ingestion-consumer/src/batcher/mod.rs
@@ -1,11 +1,12 @@
//! The worker batcher: the dispatch orchestration behind one boundary.
//!
//! The consumer loop submits one [`Accumulator`] per poll, in poll order, and
-//! receives [`GroupCompletion`]s back, one per partition and generation of each
-//! settled request. Everything in between is an implementation detail of this
-//! module: the state machine queues each key's messages, packs ready keys into
-//! requests for free worker slots, places each request on a worker as it is
-//! sent, and retries failures; one task drives it and performs its effects. No
+//! receives [`BatcherCompletion`]s back: one per partition and generation of
+//! each settled request, and one per revoke. Everything in between is an
+//! implementation detail of this module: the state machine queues each key's
+//! messages, packs ready keys into requests for free worker slots, places each
+//! request on a worker as it is sent, and retries failures; one task drives it
+//! and performs its effects. No
//! batch identity crosses the boundary: the consumer correlates completions by
//! partition and offset.
//!
@@ -14,6 +15,7 @@
//! are reported on an error channel; the consumer turns them into a process
//! failure, so the failure decision stays in the consumer loop.
+mod drains;
mod driver;
pub mod governor;
pub mod in_flight;
@@ -44,10 +46,18 @@ use crate::order_sentinel::KeyOrderSentinel;
use crate::types::Accumulator;
use crate::worker_registry::WorkerId;
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct RevokeId(pub u64);
+
+pub enum BatcherCompletion {
+ /// One partition and generation of a settled request.
+ Group(GroupCompletion),
+ Drained(RevokeId),
+}
+
/// The batcher's output channels, held by the consumer loop.
pub struct BatcherOutputs {
- /// One completion per partition and generation of each settled request.
- pub completions: mpsc::UnboundedReceiver<GroupCompletion>,
+ pub completions: mpsc::UnboundedReceiver<BatcherCompletion>,
/// Fatal orchestration failures.
pub errors: mpsc::UnboundedReceiver<String>,
}
@@ -73,9 +83,12 @@ impl Batcher {
self.driver.key_order_sentinel()
}
- /// The batcher's half of the consumer's revocation hook.
- pub fn revoker(&self) -> Revoker {
- Revoker(self.driver.revoke_sender())
+ /// Drops the partitions' queued messages, also those of polls submitted
+ /// before this call, and the messages that come back unfinished or failed.
+ /// Answers with [`BatcherCompletion::Drained`] after the last `Group` of
+ /// the partitions.
+ pub fn revoke(&self, revoke: RevokeId, partitions: &[(String, i32)]) {
+ self.driver.revoke(revoke, partitions);
}
/// Submit one poll's demuxed groups. Call on the consumer loop, in poll
@@ -142,16 +155,6 @@ impl BatcherObserver {
}
}
-/// Purges revoked partitions from the batcher. Purged messages never
-/// complete.
-pub struct Revoker(driver::RevokeSender);
-
-impl Revoker {
- pub fn purge_revoked(&self, partitions: &[(String, i32)]) {
- self.0.purge_revoked(partitions);
- }
-}
-
pub(crate) fn make_batch_id() -> String {
let ts = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
diff --git a/rust/ingestion-consumer/src/batcher/state_machine.rs b/rust/ingestion-consumer/src/batcher/state_machine.rs
index 6acf2fc5e58f..96eb1d068774 100644
--- a/rust/ingestion-consumer/src/batcher/state_machine.rs
+++ b/rust/ingestion-consumer/src/batcher/state_machine.rs
@@ -5,9 +5,9 @@
//!
//! The caller performs one action's effects in this order: order-sentinel
//! ACKs before evictions, then the sends in the order given, which is each
-//! key's send order. `next_wakeup` replaces the previous wakeup; `None` needs
-//! no timer. `fatal` means the state machine failed and the process must
-//! exit and replay.
+//! key's send order, and `drained` after `completions`. `next_wakeup`
+//! replaces the previous wakeup; `None` needs no timer. `fatal` means the
+//! state machine failed and the process must exit and replay.
//!
//! When work is pending, nothing is in flight, and no message was accepted
//! for `stall_timeout`, the state machine fails, so a wedged batcher restarts
@@ -21,6 +21,7 @@ use std::time::{Duration, Instant};
use common_kafka_consumer::{GroupCompletion, Offset, Partition};
use metrics::{counter, gauge, histogram};
+use super::drains::PartitionDrains;
use super::governor::{Adaptation, Governor, GovernorSettings, GovernorSnapshot};
use super::in_flight::{InFlightRequest, InFlightRequests, RequestId, SentMessage};
use super::key_queues::{KeyQueues, KeyRun, Settlement};
@@ -28,6 +29,7 @@ use super::packer::Packer;
use super::request::{Request, RequestClass};
use super::retry_policy::{RetryPolicy, RetryReason};
use super::worker_assigner::{WorkerAssigner, WorkerInFlight};
+use super::RevokeId;
use crate::order_sentinel::ack_mark;
use crate::types::SerializedKafkaMessage;
use crate::worker_registry::WorkerId;
@@ -73,6 +75,7 @@ pub struct WorkerOutcome {
pub struct Effects {
pub sends: Vec<SendRequest>,
pub completions: Vec<GroupCompletion>,
+ pub drained: Vec<RevokeId>,
pub key_acks: Vec<KeyAck>,
pub evicted_keys: Vec<Arc<str>>,
pub worker_outcomes: Vec<WorkerOutcome>,
@@ -143,6 +146,7 @@ impl BatcherStateMachine {
soft_budget: None,
governor,
reconnecting: HashMap::new(),
+ drains: PartitionDrains::default(),
}))
}
@@ -245,13 +249,16 @@ impl BatcherStateMachine {
/// The new owner replays the revoked partitions, so their pending
/// messages drop. Runs already in flight finish; their messages of revoked
- /// partitions that come back unfinished or failed drop too.
+ /// partitions that come back unfinished or failed drop too. `revoke`
+ /// comes back in [`Effects::drained`] once no request in flight carries
+ /// the partitions.
pub fn on_partitions_revoked(
self,
now: Instant,
+ revoke: RevokeId,
partitions: &[(String, i32)],
) -> (Self, Effects) {
- self.act(|active, _| active.on_partitions_revoked(now, partitions))
+ self.act(|active, _| active.on_partitions_revoked(now, revoke, partitions))
}
/// Takes no new groups from now on, and stops once nothing is pending or
@@ -350,6 +357,7 @@ pub struct ActiveState {
/// may wait it out and is no RTT sample. The first such request to settle
/// clears the mark.
reconnecting: HashMap<WorkerId, Instant>,
+ drains: PartitionDrains,
}
impl ActiveState {
@@ -532,12 +540,16 @@ impl ActiveState {
fn on_partitions_revoked(
&mut self,
now: Instant,
+ revoke: RevokeId,
partitions: &[(String, i32)],
) -> Result<Effects, String> {
let mut effects = Effects {
evicted_keys: self.keys.purge(partitions),
..Effects::default()
};
+ // After the purge no new request carries the partitions.
+ self.drains
+ .on_revoked(revoke, self.in_flight.carrying(partitions));
self.finish(now, &mut effects)?;
if self.keys.has_ready() {
effects.next_wakeup = Some(now);
@@ -554,6 +566,7 @@ impl ActiveState {
.in_flight
.take(request)
.ok_or_else(|| format!("response for unknown request {request:?}"))?;
+ self.drains.on_request_settled(request);
if self.assigner.release(&sent.worker, sent.message_count) {
effects.idle_workers.push(sent.worker.clone());
}
@@ -706,6 +719,7 @@ impl ActiveState {
}
fn finish(&mut self, now: Instant, effects: &mut Effects) -> Result<(), String> {
+ effects.drained.extend(self.drains.take_drained());
// A stalled action and a revoke skip `pack`, so without this a due
// retry would set the next wakeup at or before `now`.
self.keys.promote_due(now);
@@ -1185,7 +1199,8 @@ mod tests {
);
let late = now + BUSY_DELAY * 3;
- let (_, effects) = batcher.on_partitions_revoked(late, &[("events".to_string(), 9)]);
+ let (_, effects) =
+ batcher.on_partitions_revoked(late, REVOKE, &[("events".to_string(), 9)]);
assert_eq!(effects.next_wakeup, Some(late));
}
@@ -1246,7 +1261,8 @@ mod tests {
let (batcher, effects) = batcher.on_groups(now, &pool(&[]), 0, vec![run("a", &[1])]);
assert!(effects.sends.is_empty());
- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, effects) =
+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
assert!(effects.sends.is_empty());
assert_eq!(effects.evicted_keys, vec![Arc::<str>::from("a")]);
assert_eq!(batcher.pending_messages(), 0);
@@ -1710,7 +1726,8 @@ mod tests {
let (batcher, effects) = batcher.on_groups(now, &workers, 0, vec![run("a", &[1])]);
assert!(effects.sends.is_empty());
- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, effects) =
+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
assert_eq!(effects.evicted_keys, vec![Arc::<str>::from("a")]);
assert_eq!(batcher.pending_messages(), 0);
@@ -1882,7 +1899,8 @@ mod tests {
let (batcher, effects) = batcher.on_groups(now, &workers, 0, vec![later]);
assert!(effects.sends.is_empty());
- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, effects) =
+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
assert!(effects.evicted_keys.is_empty(), "a's run is in flight");
let (batcher, _) =
batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);
@@ -1910,7 +1928,7 @@ mod tests {
vec![run("a", &[1, 2])],
);
let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));
- let (batcher, _) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
// The new owner of the partition replays the uncommitted offsets.
let mut replay = run("a", &[1, 2, 3]);
for message in &mut replay.messages {
@@ -2209,7 +2227,8 @@ mod tests {
let (batcher, _) =
batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);
- let (batcher, effects) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, effects) =
+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
assert_eq!(effects.next_wakeup, Some(now), "a is ready again at once");
let (_, effects) = batcher.on_wakeup(now, &workers);
assert!(!effects.sends[0].class.replay);
@@ -2258,7 +2277,7 @@ mod tests {
let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);
let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));
let (batcher, _) = batcher.on_shutdown(now, &workers);
- let (batcher, _) = batcher.on_partitions_revoked(now, &[("events".to_string(), 0)]);
+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
assert!(
matches!(batcher, BatcherStateMachine::Draining(_)),
"a is in flight"
@@ -2271,4 +2290,211 @@ mod tests {
assert!(effects.completions.is_empty());
assert_eq!(effects.next_wakeup, None);
}
+
+ #[test]
+ fn a_partial_response_of_a_draining_partition_completes_only_what_was_accepted() {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = packing_batcher(3, Duration::ZERO, 4, now).on_groups(
+ now,
+ &workers,
+ 0,
+ vec![run("a", &[1, 2, 3])],
+ );
+ let request = effects.sends[0].request;
+ let (batcher, effects) =
+ batcher.on_partitions_revoked(now, REVOKE, &[("events".to_string(), 0)]);
+ assert!(effects.drained.is_empty(), "a is in flight");
+
+ let (_, effects) =
+ batcher.on_request_partial(now, &workers, request, 2, vec![message("a", 0, 2)]);
+ assert_eq!(effects.completions[0].offsets, vec![Offset(1), Offset(3)]);
+ assert!(
+ effects.sends.is_empty(),
+ "the unfinished offset is not resent"
+ );
+ assert_eq!(effects.drained, vec![REVOKE]);
+ }
+
+ const REVOKE: RevokeId = RevokeId(1);
+ const OTHER_REVOKE: RevokeId = RevokeId(2);
+
+ fn partition(number: i32) -> Vec<(String, i32)> {
+ vec![("events".to_string(), number)]
+ }
+
+ fn key_run(key: &str, messages: &[(i32, i64)]) -> KeyRun {
+ KeyRun {
+ routing_key: key.into(),
+ messages: messages
+ .iter()
+ .map(|&(partition, offset)| message(key, partition, offset))
+ .collect(),
+ }
+ }
+
+ fn request_of(effects: &Effects, key: &str) -> RequestId {
+ effects
+ .sends
+ .iter()
+ .find(|send| &*send.runs[0].routing_key == key)
+ .expect("a request for the key")
+ .request
+ }
+
+ fn completed_offsets(effects: &Effects) -> Vec<i64> {
+ effects
+ .completions
+ .iter()
+ .flat_map(|completion| completion.offsets.iter().map(|offset| offset.0))
+ .collect()
+ }
+
+ #[test]
+ fn a_drain_waits_only_for_the_requests_carrying_its_partitions() {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = batcher(4, now).on_groups(
+ now,
+ &workers,
+ 0,
+ vec![
+ key_run("a", &[(0, 1)]),
+ key_run("b", &[(1, 7)]),
+ key_run("c", &[(1, 8)]),
+ ],
+ );
+ let (on_p0, on_p1) = (request_of(&effects, "a"), request_of(&effects, "b"));
+ let (batcher, effects) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));
+ assert!(effects.drained.is_empty());
+
+ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, on_p1, 1);
+ assert!(
+ effects.drained.is_empty(),
+ "partition 0's request is in flight"
+ );
+
+ let (_, effects) = batcher.on_request_succeeded(now, &workers, on_p0, 1);
+ assert_eq!(
+ effects.drained,
+ vec![REVOKE],
+ "c's request for partition 1 is still in flight"
+ );
+ }
+
+ #[test]
+ fn a_failed_run_spanning_a_draining_partition_ends_the_drain_and_replays_only_the_kept_partition(
+ ) {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = packing_batcher(4, Duration::ZERO, 4, now).on_groups(
+ now,
+ &workers,
+ 0,
+ vec![key_run("a", &[(0, 1), (1, 7)])],
+ );
+ let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));
+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));
+
+ let (batcher, effects) =
+ batcher.on_request_failed(now, &workers, request, FailureCause::Fault, messages);
+ assert_eq!(
+ effects.drained,
+ vec![REVOKE],
+ "partition 1's replay is not awaited"
+ );
+
+ let (_, effects) = batcher.on_wakeup(now + FAULT_DELAY, &workers);
+ assert_eq!(shape(&effects.sends[0]), vec![("a", vec![7])]);
+ }
+
+ #[test]
+ fn a_partial_response_spanning_a_draining_partition_resends_only_the_kept_partition() {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = packing_batcher(4, Duration::ZERO, 4, now).on_groups(
+ now,
+ &workers,
+ 0,
+ vec![key_run("a", &[(0, 1), (1, 7), (0, 2), (1, 8)])],
+ );
+ let request = effects.sends[0].request;
+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));
+
+ let (_, effects) = batcher.on_request_partial(
+ now,
+ &workers,
+ request,
+ 2,
+ vec![message("a", 0, 2), message("a", 1, 8)],
+ );
+ assert_eq!(completed_offsets(&effects), vec![1, 7]);
+ assert_eq!(shape(&effects.sends[0]), vec![("a", vec![8])]);
+ assert_eq!(effects.drained, vec![REVOKE]);
+ }
+
+ #[test]
+ fn two_drains_each_end_when_their_own_requests_settle() {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = batcher(4, now).on_groups(
+ now,
+ &workers,
+ 0,
+ vec![key_run("a", &[(0, 1)]), key_run("b", &[(1, 7)])],
+ );
+ let (on_p0, on_p1) = (request_of(&effects, "a"), request_of(&effects, "b"));
+ let (batcher, _) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));
+ let (batcher, _) = batcher.on_partitions_revoked(now, OTHER_REVOKE, &partition(1));
+
+ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, on_p1, 1);
+ assert_eq!(effects.drained, vec![OTHER_REVOKE]);
+
+ let (_, effects) = batcher.on_request_succeeded(now, &workers, on_p0, 1);
+ assert_eq!(effects.drained, vec![REVOKE]);
+ }
+
+ fn queued_without_a_worker(now: Instant) -> BatcherStateMachine {
+ let (batcher, _) = batcher(4, now).on_groups(now, &pool(&[]), 0, vec![run("a", &[1])]);
+ batcher
+ }
+
+ fn waiting_for_a_retry(now: Instant) -> BatcherStateMachine {
+ let workers = pool(&["w"]);
+ let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);
+ let (request, messages) = (effects.sends[0].request, sent_messages(&effects.sends[0]));
+ let (batcher, _) =
+ batcher.on_request_failed(now, &workers, request, FailureCause::Busy, messages);
+ batcher
+ }
+
+ #[rstest]
+ #[case::queued_without_a_worker(queued_without_a_worker)]
+ #[case::waiting_for_a_retry(waiting_for_a_retry)]
+ fn a_drain_with_nothing_in_flight_ends_at_once_and_its_messages_are_never_sent(
+ #[case] setup: fn(Instant) -> BatcherStateMachine,
+ ) {
+ let now = Instant::now();
+ let (batcher, effects) = setup(now).on_partitions_revoked(now, REVOKE, &partition(0));
+ assert_eq!(effects.drained, vec![REVOKE]);
+
+ let (_, effects) = batcher.on_wakeup(now + FAULT_DELAY, &pool(&["w"]));
+ assert!(effects.sends.is_empty());
+ }
+
+ #[test]
+ fn a_drain_during_shutdown_ends_with_the_last_request() {
+ let now = Instant::now();
+ let workers = pool(&["w"]);
+ let (batcher, effects) = batcher(4, now).on_groups(now, &workers, 0, vec![run("a", &[1])]);
+ let request = effects.sends[0].request;
+ let (batcher, _) = batcher.on_shutdown(now, &workers);
+ let (batcher, effects) = batcher.on_partitions_revoked(now, REVOKE, &partition(0));
+ assert!(effects.drained.is_empty());
+
+ let (batcher, effects) = batcher.on_request_succeeded(now, &workers, request, 1);
+ assert!(matches!(batcher, BatcherStateMachine::Stopped));
+ assert_eq!(completed_offsets(&effects), vec![1]);
+ assert_eq!(effects.drained, vec![REVOKE]);
+ }
}
diff --git a/rust/ingestion-consumer/src/config.rs b/rust/ingestion-consumer/src/config.rs
index 40051f022d43..d5b8bd17f38a 100644
--- a/rust/ingestion-consumer/src/config.rs
+++ b/rust/ingestion-consumer/src/config.rs
@@ -18,6 +18,14 @@ use crate::partition_budget::BudgetLimits;
use crate::routing::{Router, RoutingStrategy};
use common_kafka_consumer::config::ConsumerConfigBuilder;
+/// The consumer loop must report healthy within this, or the lifecycle
+/// manager restarts the process.
+pub const LIVENESS_DEADLINE: Duration = Duration::from_secs(60);
+
+/// How long a shutdown waits for the consumer loop to drain its in-flight
+/// work before the process exits.
+pub const GRACEFUL_SHUTDOWN: Duration = Duration::from_secs(60);
+
/// Configuration for the ingestion consumer. Env var names and defaults match
/// the Node.js ingestion consumer, so both read the same ConfigMap.
/// `KAFKA_CONSUMER_*` vars also apply as rdkafka overrides; see
@@ -221,6 +229,17 @@ pub struct Config {
#[envconfig(from = "CONSUMER_COMMIT_INTERVAL_MS", default = "500")]
pub consumer_commit_interval_ms: u64,
+ /// How long a revoke waits for the revoked partitions' in-flight messages
+ /// before it commits them and hands the partitions to their new owner
+ /// (milliseconds). The commit gets the same time again. librdkafka fetches
+ /// no partition while it waits. `0` (default) hands them over at once,
+ /// and the new owner processes their in-flight messages again. At least
+ /// twice `INGESTION_REQUEST_SOFT_BUDGET_MS` when that is set, and at most
+ /// half the smallest of the liveness deadline, the graceful shutdown, and
+ /// `KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS`.
+ #[envconfig(from = "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", default = "0")]
+ pub consumer_revoke_drain_timeout_ms: u64,
+
/// Uncommitted events at which a partition's fetch pauses. Required.
#[envconfig(from = "CONSUMER_PARTITION_BUDGET_EVENTS", default = "0")]
pub consumer_partition_budget_events: u64,
@@ -640,6 +659,48 @@ impl Config {
}))
}
+ /// A budgeted request comes back within one soft budget. A drain shorter
+ /// than two soft budgets can time out on requests that are about to finish.
+ const MIN_REVOKE_DRAIN_SOFT_BUDGETS: u64 = 2;
+
+ pub fn revoke_drain_timeout(&self) -> Result<Option<Duration>, String> {
+ let timeout_ms = self.consumer_revoke_drain_timeout_ms;
+ if timeout_ms == 0 {
+ return Ok(None);
+ }
+ let floor_ms = self
+ .request_soft_budget_ms
+ .saturating_mul(Self::MIN_REVOKE_DRAIN_SOFT_BUDGETS);
+ if timeout_ms < floor_ms {
+ return Err(format!(
+ "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS must be 0 or at least {} times \
+ INGESTION_REQUEST_SOFT_BUDGET_MS ({floor_ms})",
+ Self::MIN_REVOKE_DRAIN_SOFT_BUDGETS
+ ));
+ }
+ // The drain's commit phase neither polls Kafka nor reports healthy for
+ // up to the timeout, a drain at shutdown adds that phase to the
+ // graceful budget, and wait plus commit must fit the broker's
+ // rebalance timeout (max.poll.interval.ms).
+ let ceiling_ms = [
+ LIVENESS_DEADLINE.as_millis() as u64,
+ GRACEFUL_SHUTDOWN.as_millis() as u64,
+ u64::from(self.kafka_consumer_max_poll_interval_ms),
+ ]
+ .into_iter()
+ .min()
+ .expect("three candidates")
+ / 2;
+ if timeout_ms > ceiling_ms {
+ return Err(format!(
+ "CONSUMER_REVOKE_DRAIN_TIMEOUT_MS must be at most half the smallest of the \
+ liveness deadline, the graceful shutdown, and \
+ KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS ({ceiling_ms})"
+ ));
+ }
+ Ok(Some(Duration::from_millis(timeout_ms)))
+ }
+
pub fn worker_urls(&self) -> Vec<String> {
self.worker_addresses
.split(',')
@@ -779,6 +840,28 @@ mod tests {
assert!(config(vars).request_budget().is_err());
}
+ struct DrainTimeoutCase {
+ vars: &'static [(&'static str, &'static str)],
+ expected: Result<Option<Duration>, ()>,
+ }
+
+ #[rstest]
+ #[case::off(DrainTimeoutCase { vars: &[("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Ok(None) })]
+ #[case::no_soft_budget(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "100")], expected: Ok(Some(Duration::from_millis(100))) })]
+ #[case::at_two_soft_budgets(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "6000"), ("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Ok(Some(Duration::from_millis(6000))) })]
+ #[case::below_two_soft_budgets(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "5999"), ("INGESTION_REQUEST_SOFT_BUDGET_MS", "3000")], expected: Err(()) })]
+ #[case::at_half_the_liveness_deadline(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "30000")], expected: Ok(Some(Duration::from_millis(30000))) })]
+ #[case::above_half_the_liveness_deadline(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "30001")], expected: Err(()) })]
+ #[case::above_half_a_short_poll_interval(DrainTimeoutCase { vars: &[("CONSUMER_REVOKE_DRAIN_TIMEOUT_MS", "10001"), ("KAFKA_CONSUMER_MAX_POLL_INTERVAL_MS", "20000")], expected: Err(()) })]
+ fn the_revoke_drain_timeout_is_off_or_between_two_soft_budgets_and_half_the_tightest_budget(
+ #[case] case: DrainTimeoutCase,
+ ) {
+ assert_eq!(
+ config(case.vars).revoke_drain_timeout().map_err(drop),
+ case.expected
+ );
+ }
+
struct LimitsCase {
vars: &'static [(&'static str, &'static str)],
expected_valid: bool,
diff --git a/rust/ingestion-consumer/src/consumer.rs b/rust/ingestion-consumer/src/consumer.rs
index b1af291221db..c37cbf6c861a 100644
--- a/rust/ingestion-consumer/src/consumer.rs
+++ b/rust/ingestion-consumer/src/consumer.rs
@@ -1,9 +1,9 @@
use std::collections::HashMap;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use std::time::Duration;
use common_kafka_consumer::{
- AssignmentEpoch, GroupCompletion, Offset, Partition, TopicOffsetLedger, TopicPartition,
+ AssignmentEpoch, Offset, Partition, TopicOffsetLedger, TopicPartition,
};
use lifecycle::Handle;
use metrics::{counter, gauge, histogram};
@@ -11,11 +11,10 @@ use rdkafka::consumer::{CommitMode, Consumer, StreamConsumer};
use rdkafka::error::KafkaError;
use rdkafka::message::BorrowedMessage;
use rdkafka::TopicPartitionList;
-use tokio::sync::mpsc;
use tokio::time::MissedTickBehavior;
use tracing::{error, info, warn};
-use crate::batcher::{make_batch_id, Batcher, BatcherObserver, BatcherOutputs, Revoker};
+use crate::batcher::{make_batch_id, Batcher, BatcherCompletion, BatcherObserver, BatcherOutputs};
use crate::charge_limit::ChargeLimit;
use crate::config::Config;
use crate::debug_recorder::{record_if, CommitOffset, DebugEventKind, DebugRecorder};
@@ -27,62 +26,16 @@ use crate::polling::{
debug_partition_offsets, debug_partition_spans, BatchStats, CollectedBatch, ContinuousPolling,
InFlightPoll, PartitionDeliveries, Parts, PollLimits, PollingStrategy, Step,
};
-use crate::rebalance::{partition_error, PartitionHook, RebalanceContext};
+use crate::rebalance::{
+ partition_error, partition_names, Drain, Event, RebalanceContext, Unassign,
+};
use crate::settlement::{Commit, GroupSettlement, Settlement};
-fn revoke_hook(
- revoker: Revoker,
- settlement: Arc<GroupSettlement>,
- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,
-) -> PartitionHook {
- Box::new(move |partitions| {
- settlement.forget_partitions(partitions);
- let pairs: Vec<(String, i32)> = partitions
- .iter()
- .map(|tp| (tp.topic.clone(), tp.partition))
- .collect();
- revoker.purge_revoked(&pairs);
- revoked_partitions
- .lock()
- .unwrap()
- .extend_from_slice(partitions);
- })
-}
-
-fn assign_hook(
- settlement: Arc<GroupSettlement>,
- assignment_epoch: AssignmentEpoch,
- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,
-) -> PartitionHook {
- Box::new(move |partitions| {
- // Also covers a loss with no revoke callback (an error rebalance, a fenced member).
- settlement.forget_partitions(partitions);
- assignment_epoch.bump();
- revoked_partitions
- .lock()
- .unwrap()
- .extend_from_slice(partitions);
- })
-}
-
-fn wire_rebalance_hooks(
- context: &RebalanceContext,
- revoker: Revoker,
- settlement: &Arc<GroupSettlement>,
- assignment_epoch: AssignmentEpoch,
-) -> Arc<Mutex<Vec<TopicPartition>>> {
- let revoked_partitions = Arc::new(Mutex::new(Vec::new()));
- context.set_on_revoke(revoke_hook(
- revoker,
- Arc::clone(settlement),
- Arc::clone(&revoked_partitions),
- ));
- context.set_on_assign(assign_hook(
- Arc::clone(settlement),
- assignment_epoch,
- Arc::clone(&revoked_partitions),
- ));
- revoked_partitions
+fn batcher_partitions(partitions: &[TopicPartition]) -> Vec<(String, i32)> {
+ partitions
+ .iter()
+ .map(|tp| (tp.topic.clone(), tp.partition))
+ .collect()
}
fn commit_tick(commit_interval: Duration) -> Duration {
@@ -107,6 +60,7 @@ pub struct IngestionConsumerOptions {
pub group_id: String,
pub debug_recorder: Option<Arc<DebugRecorder>>,
pub commit_interval: Duration,
+ pub revoke_drain_timeout: Option<Duration>,
pub ledger: Arc<TopicOffsetLedger>,
pub partition_budget: BudgetLimits,
}
@@ -130,9 +84,8 @@ pub struct IngestionConsumer {
group_id: String,
debug_recorder: Option<Arc<DebugRecorder>>,
topic_offset_ledger: Arc<TopicOffsetLedger>,
- /// Partitions revoked since the loop last looked, fed by the rebalance
- /// callback.
- revoked_partitions: Arc<Mutex<Vec<TopicPartition>>>,
+ assignment_epoch: AssignmentEpoch,
+ revoke_drain_timeout: Option<Duration>,
settlement: Arc<GroupSettlement>,
}
@@ -154,17 +107,12 @@ impl IngestionConsumer {
options.commit_interval,
options.partition_budget,
));
- let revoked_partitions = wire_rebalance_hooks(
- consumer.context(),
- batcher.revoker(),
- &settlement,
- transport.assignment_epoch(),
- );
let consumer = Arc::new(consumer);
Self {
debug_recorder: options.debug_recorder,
topic_offset_ledger,
- revoked_partitions,
+ assignment_epoch: transport.assignment_epoch(),
+ revoke_drain_timeout: options.revoke_drain_timeout,
settlement,
consumer,
batcher,
@@ -217,21 +165,17 @@ impl IngestionConsumer {
let in_flight_limit = config
.in_flight_limit()
.map_err(|err| anyhow::anyhow!("invalid in-flight limit: {err}"))?;
+ let revoke_drain_timeout = config
+ .revoke_drain_timeout()
+ .map_err(|err| anyhow::anyhow!("invalid revoke drain timeout: {err}"))?;
let commit_interval = Duration::from_millis(config.consumer_commit_interval_ms);
let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(
Arc::clone(&topic_offset_ledger),
commit_interval,
partition_budget,
));
- let context = RebalanceContext::new();
- let revoked_partitions = wire_rebalance_hooks(
- &context,
- batcher.revoker(),
- &settlement,
- transport.assignment_epoch(),
- );
let consumer: StreamConsumer<RebalanceContext> =
- client_config.create_with_context(context)?;
+ client_config.create_with_context(RebalanceContext::new())?;
consumer.subscribe(&[&config.ingestion_consumer_consume_topic])?;
let consumer = Arc::new(consumer);
@@ -249,7 +193,8 @@ impl IngestionConsumer {
consumer,
debug_recorder,
topic_offset_ledger,
- revoked_partitions,
+ assignment_epoch: transport.assignment_epoch(),
+ revoke_drain_timeout,
settlement,
batcher,
outputs: Some(outputs),
@@ -273,10 +218,7 @@ impl IngestionConsumer {
/// First waits until the statically configured workers are ready.
pub async fn process(mut self) {
let _guard = self.handle.process_scope();
- let BatcherOutputs {
- completions,
- errors,
- } = self.outputs.take().expect("process is called once");
+ let outputs = self.outputs.take().expect("process is called once");
info!("Waiting for workers to be ready");
if let Err(err) = self
@@ -306,29 +248,44 @@ impl IngestionConsumer {
group_id: self.group_id.clone(),
};
let poller = ContinuousPolling::new(parts, self.in_flight_limit);
- self.run(poller, completions, errors).await;
+ self.run(poller, outputs).await;
if let Err(err) = self.commit_all() {
warn!(error = %err, "Could not submit pending frontiers on exit");
}
info!("Consumer loop stopped");
}
- async fn run<P: PollingStrategy>(
- &self,
- mut poller: P,
- mut completions: mpsc::UnboundedReceiver<GroupCompletion>,
- mut errors: mpsc::UnboundedReceiver<String>,
- ) {
+ fn rebalance(&self) -> &RebalanceContext {
+ self.consumer.context()
+ }
+
+ async fn run<P: PollingStrategy>(&self, mut poller: P, outputs: BatcherOutputs) {
+ self.rebalance().on_started(self.revoke_drain_timeout);
+ self.run_until_stopped(&mut poller, outputs).await;
+ let drain = self.rebalance().on_stopped();
+ self.apply_rebalance_events(&mut poller);
+ if let Err(err) = self.finish_drain(&mut poller, drain, "stopped").await {
+ warn!(error = %err, "Could not commit the revoked partitions on exit");
+ }
+ }
+
+ async fn run_until_stopped<P: PollingStrategy>(&self, poller: &mut P, outputs: BatcherOutputs) {
+ let BatcherOutputs {
+ mut completions,
+ mut errors,
+ } = outputs;
let mut tick = tokio::time::interval(self.commit_tick);
tick.set_missed_tick_behavior(MissedTickBehavior::Delay);
let poll_deadline = tokio::time::sleep(Duration::ZERO);
tokio::pin!(poll_deadline);
+ let drain_deadline = tokio::time::sleep(Duration::ZERO);
+ tokio::pin!(drain_deadline);
let mut accepting_new_batches = true;
loop {
let mut submitted = false;
while let Some(step) = poller.next_step(tokio::time::Instant::now()) {
submitted |= matches!(step, Step::Submit(_));
- if let Err(failure) = self.carry_out(step, &mut poller) {
+ if let Err(failure) = self.carry_out(step, poller) {
match failure {
Stopped::Drained => {}
Stopped::Commit(err) => self.fail_commit(err),
@@ -350,6 +307,12 @@ impl IngestionConsumer {
poll_deadline.as_mut().reset(deadline);
}
}
+ let drain_until = self.rebalance().drain_deadline();
+ if let Some(until) = drain_until {
+ if drain_deadline.deadline() != until {
+ drain_deadline.as_mut().reset(until);
+ }
+ }
let reading = poller.reading();
let woke = tokio::select! {
_ = self.handle.shutdown_recv(), if accepting_new_batches => Woke::Shutdown,
@@ -357,22 +320,30 @@ impl IngestionConsumer {
failure = errors.recv() => Woke::Error(failure),
_ = tick.tick() => Woke::Tick,
_ = &mut poll_deadline, if deadline.is_some() => Woke::Deadline,
+ _ = &mut drain_deadline, if drain_until.is_some() => Woke::DrainTimedOut,
+ _ = self.rebalance().wait_for_rebalance() => Woke::Rebalanced,
received = self.consumer.recv(), if reading => Woke::Received(received),
};
- // The rebalance callbacks run inside `recv`, also inside one the
- // select dropped, so the poller sees each revoke before it acts
- // on anything that woke the loop.
- self.hand_over_revokes(&mut poller);
+ // The rebalance callbacks run inside `recv`, also in a `recv` that
+ // the select dropped. Their events apply before anything that woke
+ // the loop.
+ self.apply_rebalance_events(poller);
let failure = match woke {
Woke::Shutdown => {
info!("Shutdown signal received, draining in-flight batches");
accepting_new_batches = false;
self.batcher.begin_shutdown();
- poller.shutdown();
+ poller.on_shutdown();
None
}
- Woke::Completion(Some(completion)) => poller.completed(&completion).err(),
+ Woke::Completion(Some(BatcherCompletion::Group(completion))) => {
+ poller.on_completed(&completion).err()
+ }
+ Woke::Completion(Some(BatcherCompletion::Drained(revoke))) => {
+ let drain = self.rebalance().on_drained(revoke);
+ self.finish_drain(poller, drain, "drained").await.err()
+ }
Woke::Completion(None) => {
Some(anyhow::anyhow!("batcher completion channel closed"))
}
@@ -380,12 +351,18 @@ impl IngestionConsumer {
Woke::Error(None) => Some(anyhow::anyhow!("batcher error channel closed")),
Woke::Tick => {
self.handle.report_healthy();
- poller.ticked();
+ poller.on_ticked();
None
}
- Woke::Deadline => None,
+ Woke::Deadline | Woke::Rebalanced => None,
+ Woke::DrainTimedOut => {
+ let drain = self
+ .rebalance()
+ .on_drain_timed_out(tokio::time::Instant::now());
+ self.finish_drain(poller, drain, "timed_out").await.err()
+ }
Woke::Received(Ok(message)) => {
- poller.received(&message, tokio::time::Instant::now());
+ poller.on_received(&message, tokio::time::Instant::now());
None
}
Woke::Received(Err(err)) => self.recv_failed(err).err(),
@@ -401,7 +378,7 @@ impl IngestionConsumer {
match step {
Step::Submit(collected) => {
let poll = self.submit_poll(collected).map_err(Stopped::Batch)?;
- poller.submitted(poll);
+ poller.on_submitted(poll);
self.handle.report_healthy();
Ok(())
}
@@ -410,13 +387,90 @@ impl IngestionConsumer {
}
}
- fn hand_over_revokes<P: PollingStrategy>(&self, poller: &mut P) {
- let revoked = std::mem::take(&mut *self.revoked_partitions.lock().unwrap());
- if !revoked.is_empty() {
- poller.revoked(&revoked);
+ fn apply_rebalance_events<P: PollingStrategy>(&self, poller: &mut P) {
+ let events = self.rebalance().take_events();
+ for event in events {
+ match event {
+ Event::Assigned(assigned) => {
+ self.settlement.on_assigned(&assigned);
+ self.assignment_epoch.bump();
+ poller.on_assigned(&assigned);
+ }
+ Event::Revoked {
+ revoke,
+ partitions: revoked,
+ unassign,
+ } => {
+ self.settlement.on_revoked(&revoked);
+ self.batcher.revoke(revoke, &batcher_partitions(&revoked));
+ poller.on_revoked(&revoked);
+ unassign.apply(&*self.consumer);
+ }
+ Event::Draining {
+ revoke,
+ partitions: draining,
+ } => {
+ self.batcher.revoke(revoke, &batcher_partitions(&draining));
+ poller.on_drain_started(&draining);
+ }
+ }
}
}
+ async fn finish_drain<P: PollingStrategy>(
+ &self,
+ poller: &mut P,
+ drain: Option<Drain>,
+ outcome: &'static str,
+ ) -> anyhow::Result<()> {
+ let Some(Drain {
+ partitions: drained,
+ unassign,
+ started,
+ timeout,
+ ..
+ }) = drain
+ else {
+ return Ok(());
+ };
+ let unassign_on_drop = UnassignOnDrop {
+ consumer: Arc::clone(&self.consumer),
+ unassign: Some(unassign),
+ };
+ let committed = self.commit_drained(poller, &drained, timeout).await;
+ self.settlement.on_revoked(&drained);
+ drop(unassign_on_drop);
+ let elapsed = started.elapsed();
+ counter!("ingestion_consumer_revoke_drains_total", "outcome" => outcome).increment(1);
+ histogram!("ingestion_consumer_revoke_drain_seconds").record(elapsed.as_secs_f64());
+ info!(
+ topic_partitions = ?partition_names(&drained),
+ outcome,
+ elapsed_ms = elapsed.as_millis() as u64,
+ "Rebalance: drained partitions unassigned"
+ );
+ committed
+ }
+
+ async fn commit_drained<P: PollingStrategy>(
+ &self,
+ poller: &mut P,
+ partitions: &[TopicPartition],
+ timeout: Duration,
+ ) -> anyhow::Result<()> {
+ let Some(commit) = poller.on_drain_finished(partitions) else {
+ return Ok(());
+ };
+ for topic_partition in &commit.resume {
+ self.resume_partition(topic_partition);
+ }
+ if let Err(err) = self.commit_offsets_sync(&commit.offsets, timeout).await {
+ counter!("ingestion_consumer_revoke_commit_errors_total").increment(1);
+ warn!(error = %err, "Could not commit the revoked partitions before their release");
+ }
+ Ok(())
+ }
+
fn recv_failed(&self, err: KafkaError) -> anyhow::Result<()> {
warn!(error = %err, "Kafka recv error");
counter!("ingestion_consumer_kafka_errors_total").increment(1);
@@ -433,6 +487,34 @@ impl IngestionConsumer {
/// Submit each partition's next-to-read offset to Kafka, asynchronously.
fn commit_offsets(&self, offsets: &HashMap<TopicPartition, Offset>) -> anyhow::Result<()> {
+ let tpl = self.commit_list(offsets)?;
+ self.consumer.commit(&tpl, CommitMode::Async)?;
+ self.record_commit(offsets);
+ Ok(())
+ }
+
+ /// A commit that takes longer than `timeout` can still land after the
+ /// unassign.
+ async fn commit_offsets_sync(
+ &self,
+ offsets: &HashMap<TopicPartition, Offset>,
+ timeout: Duration,
+ ) -> anyhow::Result<()> {
+ let tpl = self.commit_list(offsets)?;
+ let consumer = Arc::clone(&self.consumer);
+ let commit = tokio::task::spawn_blocking(move || consumer.commit(&tpl, CommitMode::Sync));
+ match tokio::time::timeout(timeout, commit).await {
+ Ok(joined) => joined??,
+ Err(_) => anyhow::bail!("no answer within {timeout:?}"),
+ }
+ self.record_commit(offsets);
+ Ok(())
+ }
+
+ fn commit_list(
+ &self,
+ offsets: &HashMap<TopicPartition, Offset>,
+ ) -> anyhow::Result<TopicPartitionList> {
let mut tpl = TopicPartitionList::new();
for (topic_partition, next_to_read) in offsets {
tpl.add_partition_offset(
@@ -449,7 +531,10 @@ impl IngestionConsumer {
)
.set(next_to_read.0 as f64);
}
- self.consumer.commit(&tpl, CommitMode::Async)?;
+ Ok(tpl)
+ }
+
+ fn record_commit(&self, offsets: &HashMap<TopicPartition, Offset>) {
counter!("ingestion_consumer_offset_commits_total").increment(1);
record_if(&self.debug_recorder, || DebugEventKind::CommitSubmitted {
offsets: offsets
@@ -461,7 +546,6 @@ impl IngestionConsumer {
})
.collect(),
});
- Ok(())
}
fn submit_poll(&self, collected: CollectedBatch) -> anyhow::Result<InFlightPoll> {
@@ -601,12 +685,37 @@ impl IngestionConsumer {
}
}
+struct UnassignOnDrop {
+ consumer: Arc<StreamConsumer<RebalanceContext>>,
+ unassign: Option<Unassign>,
+}
+
+impl Drop for UnassignOnDrop {
+ fn drop(&mut self) {
+ if let Some(unassign) = self.unassign.take() {
+ unassign.apply(&*self.consumer);
+ }
+ }
+}
+
+/// A dropped task never reaches the end of `run`. The rdkafka close that
+/// follows waits for every unassign the loop still owes.
+impl Drop for IngestionConsumer {
+ fn drop(&mut self) {
+ for unassign in self.rebalance().on_dropped() {
+ unassign.apply(&*self.consumer);
+ }
+ }
+}
+
enum Woke<'a> {
Shutdown,
- Completion(Option<GroupCompletion>),
+ Completion(Option<BatcherCompletion>),
Error(Option<String>),
Tick,
Deadline,
+ DrainTimedOut,
+ Rebalanced,
Received(Result<BorrowedMessage<'a>, KafkaError>),
}
@@ -680,11 +789,34 @@ fn emit_poll_stats(
#[cfg(test)]
mod tests {
use super::*;
- use std::num::NonZeroU64;
- use common_kafka_consumer::{Charge, Offset as MessageOffset};
+ use rdkafka::config::ClientConfig;
+
+ #[tokio::test]
+ async fn a_drain_commit_cancelled_midway_still_unassigns() {
+ let consumer: StreamConsumer<RebalanceContext> = ClientConfig::new()
+ .set("bootstrap.servers", "127.0.0.1:1")
+ .set("group.id", "test")
+ .create_with_context(RebalanceContext::new())
+ .expect("a client needs no broker to be built");
+ let mut tpl = TopicPartitionList::new();
+ tpl.add_partition("events", 0);
+ consumer.assign(&tpl).expect("an assign needs no broker");
+ let consumer = Arc::new(consumer);
+ let unassign_on_drop = UnassignOnDrop {
+ consumer: Arc::clone(&consumer),
+ unassign: Some(Unassign::new(tpl, false)),
+ };
+
+ let commit = tokio::spawn(async move {
+ let _unassign_on_drop = unassign_on_drop;
+ std::future::pending::<()>().await;
+ });
+ commit.abort();
+ let _ = commit.await;
- use crate::settlement::{GroupSettlement, Pause};
+ assert_eq!(consumer.assignment().expect("assignment").count(), 0);
+ }
#[rstest::rstest]
#[case::below_a_second(200, 200)]
@@ -699,54 +831,4 @@ mod tests {
Duration::from_millis(expected_ms)
);
}
-
- #[test]
- fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {
- let ledger = Arc::new(TopicOffsetLedger::new());
- let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(
- Arc::clone(&ledger),
- Duration::from_secs(3600),
- BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),
- ));
- let epoch = AssignmentEpoch::new();
- let revoked_partitions = Arc::new(Mutex::new(Vec::new()));
- let on_assign = assign_hook(
- Arc::clone(&settlement),
- epoch.clone(),
- Arc::clone(&revoked_partitions),
- );
- let p0 = TopicPartition::new("events", 0);
- let one_event = [(
- MessageOffset(10),
- Charge {
- events: 1,
- bytes: 0,
- },
- )];
- assert_eq!(
- settlement
- .charge(&p0, 0, &one_event)
- .map(|charged| charged.pause),
- Ok(Some(Pause))
- );
- settlement.settle_run(&p0, 0, &[MessageOffset(10)]).unwrap();
-
- on_assign(std::slice::from_ref(&p0));
-
- assert!(settlement.take_all().is_none());
- assert_eq!(ledger.generation(&p0), 1);
- assert_eq!(epoch.current(), 2);
- assert_eq!(
- *revoked_partitions.lock().unwrap(),
- vec![p0.clone()],
- "the poller drops what it buffered from the lost assignment"
- );
- assert_eq!(
- settlement
- .charge(&p0, 1, &one_event)
- .map(|charged| charged.pause),
- Ok(Some(Pause)),
- "the regained partition starts unpaused"
- );
- }
}
diff --git a/rust/ingestion-consumer/src/main.rs b/rust/ingestion-consumer/src/main.rs
index c036f04e4977..ca87894f81e0 100644
--- a/rust/ingestion-consumer/src/main.rs
+++ b/rust/ingestion-consumer/src/main.rs
@@ -22,7 +22,7 @@ use tracing_subscriber::{EnvFilter, Layer};
use ingestion_consumer::aperture;
use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;
use ingestion_consumer::batcher::{Batcher, BatcherObserver};
-use ingestion_consumer::config::Config;
+use ingestion_consumer::config::{Config, GRACEFUL_SHUTDOWN, LIVENESS_DEADLINE};
use ingestion_consumer::consumer::IngestionConsumer;
use ingestion_consumer::debug_recorder::{
BatcherLoad, DebugLoad, DebugRecorder, DebugState, GovernorLoad, RoutingDebug, WorkerStatus,
@@ -56,13 +56,19 @@ fn main() -> Result<()> {
config
.in_flight_limit()
.map_err(|err| anyhow::anyhow!("invalid in-flight limit: {err}"))?;
+ config
+ .revoke_drain_timeout()
+ .map_err(|err| anyhow::anyhow!("invalid revoke drain timeout: {err}"))?;
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.context("Failed to build tokio runtime")?;
- runtime.block_on(async_main(config))
+ let result = runtime.block_on(async_main(config));
+ // A blocking commit that outlived its timeout must not hold the exit.
+ runtime.shutdown_timeout(Duration::from_secs(5));
+ result
}
async fn async_main(config: Config) -> Result<()> {
@@ -116,8 +122,8 @@ async fn async_main(config: Config) -> Result<()> {
let consumer_handle = manager.register(
"consumer",
ComponentOptions::new()
- .with_graceful_shutdown(Duration::from_secs(60))
- .with_liveness_deadline(Duration::from_secs(60))
+ .with_graceful_shutdown(GRACEFUL_SHUTDOWN)
+ .with_liveness_deadline(LIVENESS_DEADLINE)
.with_stall_threshold(3),
);
diff --git a/rust/ingestion-consumer/src/polling/continuous.rs b/rust/ingestion-consumer/src/polling/continuous.rs
index edb7d7ad1be2..97411e13300a 100644
--- a/rust/ingestion-consumer/src/polling/continuous.rs
+++ b/rust/ingestion-consumer/src/polling/continuous.rs
@@ -55,11 +55,11 @@ impl PollingStrategy for ContinuousPolling {
self.collector.deadline()
}
- fn received<M: Message>(&mut self, message: &M, now: Instant) {
+ fn on_received<M: Message>(&mut self, message: &M, now: Instant) {
self.collector.add(message, now);
}
- fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {
+ fn on_completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()> {
if (completion.accepted as usize) < completion.offsets.len() {
anyhow::bail!(
"accepted {}/{} messages — not committing offsets",
@@ -84,26 +84,40 @@ impl PollingStrategy for ContinuousPolling {
Ok(())
}
- fn ticked(&mut self) {
+ fn on_ticked(&mut self) {
self.commit_due = true;
}
- fn revoked(&mut self, topic_partitions: &[TopicPartition]) {
- let building = self.collector.remove_partitions(topic_partitions) as u64;
- self.uncommitted.revoked(topic_partitions);
- self.processed.revoked(topic_partitions);
- report_revoked(building);
+ fn on_assigned(&mut self, topic_partitions: &[TopicPartition]) {
+ self.forget_partitions(topic_partitions);
+ }
+
+ fn on_drain_started(&mut self, topic_partitions: &[TopicPartition]) {
+ report_revoked(self.collector.remove_partitions(topic_partitions) as u64);
+ }
+
+ fn on_drain_finished(&mut self, topic_partitions: &[TopicPartition]) -> Option<Commit> {
+ let commit = self
+ .settlement
+ .take_all()
+ .map(|commit| self.record_commit(commit));
+ self.on_revoked(topic_partitions);
+ commit
+ }
+
+ fn on_revoked(&mut self, topic_partitions: &[TopicPartition]) {
+ self.forget_partitions(topic_partitions);
}
- fn shutdown(&mut self) {
+ fn on_shutdown(&mut self) {
self.accepting_new_batches = false;
self.commit_due = true;
self.collector.discard();
}
- fn submitted(&mut self, poll: InFlightPoll) {
- self.uncommitted.charged(&poll);
- self.processed.submitted(&poll);
+ fn on_submitted(&mut self, poll: InFlightPoll) {
+ self.uncommitted.record_poll(&poll);
+ self.processed.record_poll(&poll);
}
fn next_step(&mut self, now: Instant) -> Option<Step> {
@@ -114,9 +128,7 @@ impl PollingStrategy for ContinuousPolling {
self.settlement.take_all()
};
if let Some(commit) = commit {
- self.uncommitted.committed(&commit);
- self.processed.committed(&commit);
- return Some(Step::Commit(commit));
+ return Some(Step::Commit(self.record_commit(commit)));
}
}
if let Some(collected) = self.collector.take_ready(now) {
@@ -126,6 +138,21 @@ impl PollingStrategy for ContinuousPolling {
}
}
+impl ContinuousPolling {
+ fn record_commit(&mut self, commit: Commit) -> Commit {
+ self.uncommitted.record_commit(&commit);
+ self.processed.record_commit(&commit);
+ commit
+ }
+
+ fn forget_partitions(&mut self, topic_partitions: &[TopicPartition]) {
+ let building = self.collector.remove_partitions(topic_partitions) as u64;
+ self.uncommitted.forget(topic_partitions);
+ self.processed.forget(topic_partitions);
+ report_revoked(building);
+ }
+}
+
struct ProcessedTimestamps {
group_id: String,
partitions: HashMap<TopicPartition, VecDeque<SubmittedSlice>>,
@@ -144,7 +171,7 @@ impl ProcessedTimestamps {
}
}
- fn submitted(&mut self, poll: &InFlightPoll) {
+ fn record_poll(&mut self, poll: &InFlightPoll) {
for (topic_partition, deliveries) in &poll.partitions {
self.partitions
.entry(topic_partition.clone())
@@ -156,7 +183,7 @@ impl ProcessedTimestamps {
}
}
- fn committed(&mut self, commit: &Commit) {
+ fn record_commit(&mut self, commit: &Commit) {
for (topic_partition, next_to_read) in &commit.offsets {
let Some(polls) = self.partitions.get_mut(topic_partition) else {
continue;
@@ -182,8 +209,8 @@ impl ProcessedTimestamps {
}
}
- fn revoked(&mut self, revoked: &[TopicPartition]) {
- for topic_partition in revoked {
+ fn forget(&mut self, topic_partitions: &[TopicPartition]) {
+ for topic_partition in topic_partitions {
self.partitions.remove(topic_partition);
}
}
@@ -205,14 +232,14 @@ impl UncommittedCharges {
self.total == Charge::ZERO
}
- /// `None` once the partition is revoked, until its next charge.
+ /// `None` once the partition is forgotten, until its next charge.
fn topic_partition(&self, partition: Partition) -> Option<&TopicPartition> {
self.partitions
.get(&partition)
.map(|(topic_partition, _)| topic_partition)
}
- fn charged(&mut self, poll: &InFlightPoll) {
+ fn record_poll(&mut self, poll: &InFlightPoll) {
for (topic_partition, &charge) in &poll.uncommitted {
let previous = self.partitions.insert(
Partition(topic_partition.partition),
@@ -222,7 +249,7 @@ impl UncommittedCharges {
}
}
- fn committed(&mut self, commit: &Commit) {
+ fn record_commit(&mut self, commit: &Commit) {
for (topic_partition, &charge) in &commit.uncommitted {
if let Some((_, partition)) = self
.partitions
@@ -234,8 +261,8 @@ impl UncommittedCharges {
}
}
- fn revoked(&mut self, revoked: &[TopicPartition]) {
- for topic_partition in revoked {
+ fn forget(&mut self, topic_partitions: &[TopicPartition]) {
+ for topic_partition in topic_partitions {
let previous = self
.partitions
.remove(&Partition(topic_partition.partition));
@@ -296,7 +323,7 @@ mod tests {
harness.receive(0, &[10, 11]);
harness.receive(1, &[20]);
- harness.poller.revoked(&[p(0)]);
+ harness.poller.on_revoked(&[p(0)]);
harness.receive(0, &[10]);
harness.after_the_batch_timeout();
@@ -339,7 +366,7 @@ mod tests {
harness.submit();
harness.complete_under(before, 0, &[0, 1]);
- harness.poller.ticked();
+ harness.poller.on_ticked();
assert!(
harness.step().is_none(),
"the old assignment's completion is stale"
@@ -350,6 +377,66 @@ mod tests {
assert_eq!(processed(), 2);
}
+ #[test]
+ fn a_started_drain_drops_the_poll_in_progress_and_settles_what_was_submitted() {
+ let mut harness = continuous(1_000);
+ harness.receive(0, &[0, 1]);
+ harness.after_the_batch_timeout();
+ harness.submit();
+ harness.receive(0, &[2]);
+
+ harness.poller.on_drain_started(&[p(0)]);
+ harness.after_the_batch_timeout();
+ assert!(
+ harness.step().is_none(),
+ "offset 2 was never charged, so the new owner reads it"
+ );
+
+ harness.complete(0, &[0, 1]);
+ assert_eq!(harness.commit(), HashMap::from([(p(0), Offset(2))]));
+ }
+
+ #[test]
+ fn a_finished_drain_commits_up_to_the_first_unprocessed_offset() {
+ let mut harness = continuous(1_000);
+ harness.receive(0, &[0, 1, 2]);
+ harness.after_the_batch_timeout();
+ harness.submit();
+
+ harness.poller.on_drain_started(&[p(0)]);
+ harness.complete(0, &[0, 2]);
+
+ assert_eq!(
+ harness.finish_drain(0),
+ HashMap::from([(p(0), Offset(1))]),
+ "offset 1 came back unfinished"
+ );
+ }
+
+ #[test]
+ fn a_finished_drain_frees_the_partitions_charge_and_ignores_its_late_completions() {
+ let mut harness = continuous(3);
+ harness.receive(0, &[0, 1, 2]);
+ harness.after_the_batch_timeout();
+ harness.submit();
+ assert!(!harness.poller.reading());
+ let drained_generation = harness.ledger.generation(&p(0));
+
+ harness.poller.on_drain_started(&[p(0)]);
+ assert_eq!(harness.finish_drain(0), HashMap::new());
+ assert!(
+ harness.poller.reading(),
+ "a drained partition holds no charge"
+ );
+
+ harness.complete_under(drained_generation, 0, &[0, 1, 2]);
+ harness.poller.on_ticked();
+ assert!(
+ harness.step().is_none(),
+ "a late completion commits nothing"
+ );
+ }
+
#[test]
fn a_revoke_frees_the_partitions_charge() {
let mut harness = continuous(3);
@@ -414,7 +501,7 @@ mod tests {
let error = harness
.poller
- .completed(&short)
+ .on_completed(&short)
.expect_err("a dropped message must not commit");
assert!(error.to_string().contains("accepted 1/2"), "{error}");
}
@@ -458,7 +545,7 @@ mod tests {
harness.submit();
harness.receive(0, &[1]);
- harness.poller.shutdown();
+ harness.poller.on_shutdown();
assert!(!harness.poller.reading());
assert_eq!(harness.poller.deadline(), None);
assert!(harness.step().is_none(), "offset 0 is still in flight");
diff --git a/rust/ingestion-consumer/src/polling/harness.rs b/rust/ingestion-consumer/src/polling/harness.rs
index bd4e7bee210d..94f4cb2bd20d 100644
--- a/rust/ingestion-consumer/src/polling/harness.rs
+++ b/rust/ingestion-consumer/src/polling/harness.rs
@@ -24,7 +24,7 @@ pub(super) struct Harness<P> {
impl<P: PollingStrategy> Harness<P> {
pub(super) fn new(make: impl FnOnce(Parts) -> P) -> Self {
let ledger = Arc::new(TopicOffsetLedger::new());
- let settlement: Arc<GroupSettlement> = Arc::new(GroupSettlement::new(
+ let settlement = Arc::new(GroupSettlement::new(
Arc::clone(&ledger),
Duration::ZERO,
BudgetLimits::unset(),
@@ -49,7 +49,7 @@ impl<P: PollingStrategy> Harness<P> {
pub(super) fn receive(&mut self, partition: i32, offsets: &[i64]) {
for &offset in offsets {
self.poller
- .received(&message(partition, offset, 10), self.now);
+ .on_received(&message(partition, offset, 10), self.now);
}
}
@@ -84,7 +84,7 @@ impl<P: PollingStrategy> Harness<P> {
)
})
.collect();
- self.poller.submitted(InFlightPoll {
+ self.poller.on_submitted(InFlightPoll {
partitions,
uncommitted,
});
@@ -98,13 +98,23 @@ impl<P: PollingStrategy> Harness<P> {
pub(super) fn complete_under(&mut self, generation: u64, partition: i32, offsets: &[i64]) {
self.poller
- .completed(&completion(generation, partition, offsets))
+ .on_completed(&completion(generation, partition, offsets))
.expect("the completion applies");
}
pub(super) fn revoke(&mut self, partition: i32) {
- self.settlement.forget_partitions(&[p(partition)]);
- self.poller.revoked(&[p(partition)]);
+ self.settlement.on_revoked(&[p(partition)]);
+ self.poller.on_revoked(&[p(partition)]);
+ }
+
+ pub(super) fn finish_drain(&mut self, partition: i32) -> HashMap<TopicPartition, Offset> {
+ let committed = self
+ .poller
+ .on_drain_finished(&[p(partition)])
+ .map(|commit| commit.offsets)
+ .unwrap_or_default();
+ self.settlement.on_revoked(&[p(partition)]);
+ committed
}
pub(super) fn commit(&mut self) -> HashMap<TopicPartition, Offset> {
diff --git a/rust/ingestion-consumer/src/polling/mod.rs b/rust/ingestion-consumer/src/polling/mod.rs
index feb998cd9979..bfe2becf71b8 100644
--- a/rust/ingestion-consumer/src/polling/mod.rs
+++ b/rust/ingestion-consumer/src/polling/mod.rs
@@ -47,22 +47,29 @@ pub(crate) trait PollingStrategy {
fn deadline(&self) -> Option<Instant>;
- fn received<M: Message>(&mut self, message: &M, now: Instant);
+ fn on_received<M: Message>(&mut self, message: &M, now: Instant);
- fn completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()>;
+ fn on_completed(&mut self, completion: &GroupCompletion) -> anyhow::Result<()>;
- fn ticked(&mut self);
+ fn on_ticked(&mut self);
+
+ fn on_assigned(&mut self, topic_partitions: &[TopicPartition]);
+
+ fn on_drain_started(&mut self, topic_partitions: &[TopicPartition]);
+
+ /// The commit holds every partition's frontier, not only the drained ones.
+ fn on_drain_finished(&mut self, topic_partitions: &[TopicPartition]) -> Option<Commit>;
/// Every message received before the revoke belongs to the old
/// assignment: the batcher purges it, or Kafka delivers it again from the
/// last commit.
- fn revoked(&mut self, topic_partitions: &[TopicPartition]);
+ fn on_revoked(&mut self, topic_partitions: &[TopicPartition]);
/// The poll in progress is not charged yet, so it is dropped and replays
/// from the last commit.
- fn shutdown(&mut self);
+ fn on_shutdown(&mut self);
- fn submitted(&mut self, poll: InFlightPoll);
+ fn on_submitted(&mut self, poll: InFlightPoll);
fn next_step(&mut self, now: Instant) -> Option<Step>;
}
diff --git a/rust/ingestion-consumer/src/rebalance.rs b/rust/ingestion-consumer/src/rebalance.rs
deleted file mode 100644
index 76d0e135aa1e..000000000000
--- a/rust/ingestion-consumer/src/rebalance.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-//! The consumer's rdkafka context: rebalance callbacks into hooks, plus rebalance counters and client statistics.
-
-use std::sync::OnceLock;
-
-use common_kafka_consumer::TopicPartition;
-use metrics::counter;
-use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext, Rebalance};
-use rdkafka::error::KafkaResult;
-use rdkafka::{ClientContext, Statistics, TopicPartitionList};
-use tracing::{error, info, warn};
-
-pub type PartitionHook = Box<dyn Fn(&[TopicPartition]) + Send + Sync>;
-
-/// Hooks run inside the callback, before the next poll delivers the new assignment.
-#[derive(Default)]
-pub struct RebalanceContext {
- on_revoke: OnceLock<PartitionHook>,
- on_assign: OnceLock<PartitionHook>,
-}
-
-impl RebalanceContext {
- pub fn new() -> Self {
- Self::default()
- }
-
- /// Settable after construction because the rdkafka consumer only hands out `&self`.
- pub fn set_on_revoke(&self, hook: PartitionHook) {
- let _ = self.on_revoke.set(hook);
- }
-
- pub fn set_on_assign(&self, hook: PartitionHook) {
- let _ = self.on_assign.set(hook);
- }
-}
-
-fn topic_partitions(tpl: &TopicPartitionList) -> Vec<TopicPartition> {
- let mut partitions: Vec<TopicPartition> = tpl
- .elements()
- .iter()
- .map(|element| TopicPartition::new(element.topic(), element.partition()))
- .collect();
- partitions.sort_unstable_by(|a, b| (&a.topic, a.partition).cmp(&(&b.topic, b.partition)));
- partitions
-}
-
-/// librdkafka's pause and resume return success when they cannot change a
-/// partition. They set the error on that partition's list element.
-pub(crate) fn partition_error(tpl: &TopicPartitionList) -> KafkaResult<()> {
- tpl.elements()
- .iter()
- .try_for_each(|element| element.error())
-}
-
-fn partition_names(partitions: &[TopicPartition]) -> Vec<String> {
- partitions
- .iter()
- .map(|tp| format!("{}:{}", tp.topic, tp.partition))
- .collect()
-}
-
-impl ClientContext for RebalanceContext {
- /// Fired on a librdkafka thread every `statistics.interval.ms`; disabled
- /// when that is 0.
- fn stats(&self, stats: Statistics) {
- crate::kafka_stats::export(&stats);
- }
-}
-
-impl ConsumerContext for RebalanceContext {
- fn pre_rebalance(&self, _consumer: &BaseConsumer<Self>, rebalance: &Rebalance) {
- match rebalance {
- Rebalance::Revoke(tpl) => {
- counter!("ingestion_consumer_rebalances_total", "event" => "revoke").increment(1);
- let partitions = topic_partitions(tpl);
- info!(
- partitions = partitions.len(),
- topic_partitions = ?partition_names(&partitions),
- "Rebalance: partitions revoked"
- );
- if let Some(hook) = self.on_revoke.get() {
- hook(&partitions);
- }
- }
- Rebalance::Assign(_) => {}
- Rebalance::Error(err) => {
- counter!("ingestion_consumer_rebalances_total", "event" => "error").increment(1);
- warn!(error = %err, "Rebalance error");
- }
- }
- }
-
- fn post_rebalance(&self, consumer: &BaseConsumer<Self>, rebalance: &Rebalance) {
- if let Rebalance::Assign(tpl) = rebalance {
- counter!("ingestion_consumer_rebalances_total", "event" => "assign").increment(1);
- let partitions = topic_partitions(tpl);
- info!(
- partitions = partitions.len(),
- topic_partitions = ?partition_names(&partitions),
- "Rebalance: partitions assigned"
- );
- // librdkafka keeps a partition's pause across a revoke and a
- // reassignment, and the assign hook forgets the budget that paused it.
- if let Err(err) = consumer.resume(tpl).and_then(|()| partition_error(tpl)) {
- error!(error = %err, "Could not resume the assigned partitions");
- }
- if let Some(hook) = self.on_assign.get() {
- hook(&partitions);
- }
- }
- }
-
- // NOTE: `ConsumerContext::commit_callback` is deliberately not implemented.
- // librdkafka only propagates a commit result to the application when a
- // conf-level `offset_commit_cb` is registered (rust-rdkafka never does) or
- // when the commit is synchronous (a replyq is attached); manual async
- // commits silently drop their result (`rd_kafka_cgrp_propagate_commit_result`).
-}
-
-#[cfg(test)]
-mod tests {
- use std::sync::{Arc, Mutex};
-
- use rdkafka::config::ClientConfig;
- use rdkafka::consumer::Consumer;
-
- use super::*;
-
- fn recording(seen: &Arc<Mutex<Vec<TopicPartition>>>) -> PartitionHook {
- let seen = Arc::clone(seen);
- Box::new(move |partitions| seen.lock().unwrap().extend_from_slice(partitions))
- }
-
- #[test]
- fn partitions_sort_by_topic_then_partition_number() {
- let mut tpl = TopicPartitionList::new();
- tpl.add_partition("overflow", 2);
- tpl.add_partition("events", 10);
- tpl.add_partition("events", 9);
-
- assert_eq!(
- partition_names(&topic_partitions(&tpl)),
- vec!["events:9", "events:10", "overflow:2"]
- );
- }
-
- #[test]
- fn each_callback_reaches_its_hook_with_the_partitions() {
- let revoked = Arc::new(Mutex::new(Vec::new()));
- let assigned = Arc::new(Mutex::new(Vec::new()));
- let context = RebalanceContext::new();
- context.set_on_revoke(recording(&revoked));
- context.set_on_assign(recording(&assigned));
- let consumer: BaseConsumer<RebalanceContext> = ClientConfig::new()
- .set("bootstrap.servers", "127.0.0.1:1")
- .set("group.id", "test")
- .create_with_context(context)
- .expect("a client needs no broker to be built");
- let mut tpl = TopicPartitionList::new();
- tpl.add_partition("events", 1);
- tpl.add_partition("events", 0);
-
- consumer
- .context()
- .pre_rebalance(&consumer, &Rebalance::Revoke(&tpl));
- consumer
- .context()
- .post_rebalance(&consumer, &Rebalance::Assign(&tpl));
-
- let expected = vec![
- TopicPartition::new("events", 0),
- TopicPartition::new("events", 1),
- ];
- assert_eq!(*revoked.lock().unwrap(), expected);
- assert_eq!(*assigned.lock().unwrap(), expected);
- }
-
- #[test]
- fn pausing_an_unassigned_partition_is_an_error() {
- let consumer: BaseConsumer = ClientConfig::new()
- .set("bootstrap.servers", "127.0.0.1:1")
- .set("group.id", "test")
- .create()
- .expect("a client needs no broker to be built");
- let mut tpl = TopicPartitionList::new();
- tpl.add_partition("events", 0);
-
- assert!(consumer.pause(&tpl).is_ok());
- assert!(partition_error(&tpl).is_err());
- }
-}
diff --git a/rust/ingestion-consumer/src/rebalance/context.rs b/rust/ingestion-consumer/src/rebalance/context.rs
new file mode 100644
index 000000000000..3f46b11afcda
--- /dev/null
+++ b/rust/ingestion-consumer/src/rebalance/context.rs
@@ -0,0 +1,262 @@
+//! The consumer's rdkafka context: rebalance callbacks into [`RebalanceState`], plus rebalance counters and client statistics.
+
+use std::sync::{Mutex, MutexGuard};
+use std::time::Duration;
+
+use common_kafka_consumer::TopicPartition;
+use metrics::counter;
+use rdkafka::consumer::{BaseConsumer, Consumer, ConsumerContext, RebalanceProtocol};
+use rdkafka::error::{KafkaResult, RDKafkaErrorCode};
+use rdkafka::types::RDKafkaRespErr;
+use rdkafka::{ClientContext, Statistics, TopicPartitionList};
+use tokio::sync::Notify;
+use tokio::time::Instant;
+use tracing::{error, info, warn};
+
+use super::state::{Drain, Event, RebalanceState, Unassign};
+use crate::batcher::RevokeId;
+
+#[derive(Default)]
+pub struct RebalanceContext {
+ rebalance_state: Mutex<RebalanceState>,
+ rebalanced: Notify,
+}
+
+impl RebalanceContext {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub(crate) fn on_started(&self, drain_timeout: Option<Duration>) {
+ self.rebalance_state().on_started(drain_timeout);
+ }
+
+ pub(crate) fn on_stopped(&self) -> Option<Drain> {
+ self.rebalance_state().on_stopped()
+ }
+
+ pub(crate) fn on_dropped(&self) -> Vec<Unassign> {
+ self.rebalance_state().on_dropped()
+ }
+
+ pub(crate) fn on_drained(&self, revoke: RevokeId) -> Option<Drain> {
+ self.rebalance_state().on_drained(revoke)
+ }
+
+ pub(crate) fn on_drain_timed_out(&self, now: Instant) -> Option<Drain> {
+ self.rebalance_state().on_drain_timed_out(now)
+ }
+
+ pub(crate) fn take_events(&self) -> Vec<Event> {
+ self.rebalance_state().take_events()
+ }
+
+ pub(crate) fn drain_deadline(&self) -> Option<Instant> {
+ self.rebalance_state().drain_deadline()
+ }
+
+ pub(crate) async fn wait_for_rebalance(&self) {
+ self.rebalanced.notified().await;
+ }
+
+ fn rebalance_state(&self) -> MutexGuard<'_, RebalanceState> {
+ self.rebalance_state.lock().unwrap()
+ }
+
+ fn on_assigned(&self, consumer: &BaseConsumer<Self>, tpl: &TopicPartitionList) {
+ counter!("ingestion_consumer_rebalances_total", "event" => "assign").increment(1);
+ let partitions = topic_partitions(tpl);
+ info!(
+ partitions = partitions.len(),
+ topic_partitions = ?partition_names(&partitions),
+ "Rebalance: partitions assigned"
+ );
+ let assigned = if cooperative(consumer) {
+ consumer.incremental_assign(tpl)
+ } else {
+ consumer.assign(tpl)
+ };
+ if let Err(err) = assigned {
+ error!(error = %err, "Could not assign the partitions");
+ }
+ // librdkafka keeps a partition's pause across a revoke and a
+ // reassignment, and the consumer forgets the budget that paused it.
+ if let Err(err) = consumer.resume(tpl).and_then(|()| partition_error(tpl)) {
+ error!(error = %err, "Could not resume the assigned partitions");
+ }
+ self.rebalance_state().on_assigned(partitions);
+ }
+
+ fn on_revoked(&self, consumer: &BaseConsumer<Self>, tpl: &TopicPartitionList) {
+ counter!("ingestion_consumer_rebalances_total", "event" => "revoke").increment(1);
+ let partitions = topic_partitions(tpl);
+ let names = partition_names(&partitions);
+ info!(
+ partitions = names.len(),
+ topic_partitions = ?names,
+ "Rebalance: partitions revoked"
+ );
+ let unassign = self.rebalance_state().on_revoked(
+ partitions,
+ Unassign::new(tpl.clone(), cooperative(consumer)),
+ consumer.assignment_lost(),
+ Instant::now(),
+ );
+ if let Some(unassign) = unassign {
+ unassign.apply(consumer);
+ }
+ }
+}
+
+fn cooperative<C: ConsumerContext>(consumer: &BaseConsumer<C>) -> bool {
+ matches!(
+ consumer.rebalance_protocol(),
+ RebalanceProtocol::Cooperative
+ )
+}
+
+fn topic_partitions(tpl: &TopicPartitionList) -> Vec<TopicPartition> {
+ let mut partitions: Vec<TopicPartition> = tpl
+ .elements()
+ .iter()
+ .map(|element| TopicPartition::new(element.topic(), element.partition()))
+ .collect();
+ partitions.sort_unstable_by(|a, b| (&a.topic, a.partition).cmp(&(&b.topic, b.partition)));
+ partitions
+}
+
+/// librdkafka's pause and resume return success when they cannot change a
+/// partition. They set the error on that partition's list element.
+pub(crate) fn partition_error(tpl: &TopicPartitionList) -> KafkaResult<()> {
+ tpl.elements()
+ .iter()
+ .try_for_each(|element| element.error())
+}
+
+pub(crate) fn partition_names(partitions: &[TopicPartition]) -> Vec<String> {
+ partitions
+ .iter()
+ .map(|tp| format!("{}:{}", tp.topic, tp.partition))
+ .collect()
+}
+
+impl ClientContext for RebalanceContext {
+ /// Fired on a librdkafka thread every `statistics.interval.ms`; disabled
+ /// when that is 0.
+ fn stats(&self, stats: Statistics) {
+ crate::kafka_stats::export(&stats);
+ }
+}
+
+impl ConsumerContext for RebalanceContext {
+ fn rebalance(
+ &self,
+ consumer: &BaseConsumer<Self>,
+ err: RDKafkaRespErr,
+ tpl: &mut TopicPartitionList,
+ ) {
+ match err {
+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS => self.on_assigned(consumer, tpl),
+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS => self.on_revoked(consumer, tpl),
+ // librdkafka only calls back with assign or revoke. This arm is a
+ // fallback for another code, and the unassign keeps the group from
+ // waiting on this member.
+ _ => {
+ warn!(error = %RDKafkaErrorCode::from(err), "Rebalance error");
+ Unassign::new(tpl.clone(), cooperative(consumer)).apply(consumer);
+ }
+ }
+ self.rebalanced.notify_one();
+ }
+
+ // NOTE: `ConsumerContext::commit_callback` is deliberately not implemented.
+ // librdkafka only propagates a commit result to the application when a
+ // conf-level `offset_commit_cb` is registered (rust-rdkafka never does) or
+ // when the commit is synchronous (a replyq is attached); manual async
+ // commits silently drop their result (`rd_kafka_cgrp_propagate_commit_result`).
+}
+
+#[cfg(test)]
+mod tests {
+ use rdkafka::config::ClientConfig;
+ use rdkafka::consumer::Consumer;
+
+ use super::*;
+
+ #[test]
+ fn partitions_sort_by_topic_then_partition_number() {
+ let mut tpl = TopicPartitionList::new();
+ tpl.add_partition("overflow", 2);
+ tpl.add_partition("events", 10);
+ tpl.add_partition("events", 9);
+
+ assert_eq!(
+ partition_names(&topic_partitions(&tpl)),
+ vec!["events:9", "events:10", "overflow:2"]
+ );
+ }
+
+ struct CallbackCase {
+ loop_running: bool,
+ revoke_queued: bool,
+ still_assigned: usize,
+ }
+
+ #[rstest::rstest]
+ #[case::loop_running(CallbackCase { loop_running: true, revoke_queued: true, still_assigned: 2 })]
+ #[case::no_loop(CallbackCase { loop_running: false, revoke_queued: false, still_assigned: 0 })]
+ fn the_loop_applies_the_unassign_while_it_runs(#[case] case: CallbackCase) {
+ let consumer: BaseConsumer<RebalanceContext> = ClientConfig::new()
+ .set("bootstrap.servers", "127.0.0.1:1")
+ .set("group.id", "test")
+ .create_with_context(RebalanceContext::new())
+ .expect("a client needs no broker to be built");
+ let mut tpl = TopicPartitionList::new();
+ tpl.add_partition("events", 1);
+ tpl.add_partition("events", 0);
+ let context = consumer.context();
+ if case.loop_running {
+ context.on_started(None);
+ }
+
+ context.rebalance(
+ &consumer,
+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS,
+ &mut tpl,
+ );
+ context.rebalance(
+ &consumer,
+ RDKafkaRespErr::RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS,
+ &mut tpl,
+ );
+
+ let partitions = vec![
+ TopicPartition::new("events", 0),
+ TopicPartition::new("events", 1),
+ ];
+ let mut expected = vec![Event::Assigned(partitions.clone())];
+ if case.revoke_queued {
+ expected.push(Event::Revoked {
+ revoke: RevokeId(1),
+ partitions,
+ unassign: Unassign::new(tpl.clone(), false),
+ });
+ }
+ assert_eq!(context.take_events(), expected);
+ assert_eq!(consumer.assignment().unwrap().count(), case.still_assigned);
+ }
+
+ #[test]
+ fn pausing_an_unassigned_partition_is_an_error() {
+ let consumer: BaseConsumer = ClientConfig::new()
+ .set("bootstrap.servers", "127.0.0.1:1")
+ .set("group.id", "test")
+ .create()
+ .expect("a client needs no broker to be built");
+ let mut tpl = TopicPartitionList::new();
+ tpl.add_partition("events", 0);
+
+ assert!(consumer.pause(&tpl).is_ok());
+ assert!(partition_error(&tpl).is_err());
+ }
+}
diff --git a/rust/ingestion-consumer/src/rebalance/mod.rs b/rust/ingestion-consumer/src/rebalance/mod.rs
new file mode 100644
index 000000000000..6052edb5706d
--- /dev/null
+++ b/rust/ingestion-consumer/src/rebalance/mod.rs
@@ -0,0 +1,6 @@
+mod context;
+mod state;
+
+pub use context::RebalanceContext;
+pub(crate) use context::{partition_error, partition_names};
+pub(crate) use state::{Drain, Event, Unassign};
diff --git a/rust/ingestion-consumer/src/rebalance/state.rs b/rust/ingestion-consumer/src/rebalance/state.rs
new file mode 100644
index 000000000000..5b4f6eb5d849
--- /dev/null
+++ b/rust/ingestion-consumer/src/rebalance/state.rs
@@ -0,0 +1,382 @@
+use std::time::Duration;
+
+use common_kafka_consumer::TopicPartition;
+use rdkafka::consumer::{Consumer, ConsumerContext};
+use rdkafka::TopicPartitionList;
+use tokio::time::Instant;
+use tracing::error;
+
+use crate::batcher::RevokeId;
+
+#[derive(Debug, PartialEq)]
+pub(crate) enum Event {
+ Assigned(Vec<TopicPartition>),
+ Revoked {
+ revoke: RevokeId,
+ partitions: Vec<TopicPartition>,
+ unassign: Unassign,
+ },
+ Draining {
+ revoke: RevokeId,
+ partitions: Vec<TopicPartition>,
+ },
+}
+
+#[derive(Debug, PartialEq)]
+pub(crate) struct Unassign {
+ partitions: TopicPartitionList,
+ cooperative: bool,
+}
+
+impl Unassign {
+ pub(crate) fn new(partitions: TopicPartitionList, cooperative: bool) -> Self {
+ Self {
+ partitions,
+ cooperative,
+ }
+ }
+
+ pub(crate) fn apply<C: ConsumerContext>(self, consumer: &impl Consumer<C>) {
+ let applied = if self.cooperative {
+ consumer.incremental_unassign(&self.partitions)
+ } else {
+ consumer.unassign()
+ };
+ if let Err(err) = applied {
+ error!(error = %err, "Could not unassign the revoked partitions");
+ }
+ }
+}
+
+pub(crate) struct Drain {
+ revoke: RevokeId,
+ pub(crate) partitions: Vec<TopicPartition>,
+ pub(crate) unassign: Unassign,
+ pub(crate) started: Instant,
+ pub(crate) timeout: Duration,
+}
+
+enum Phase {
+ Starting,
+ Consuming { drain_timeout: Option<Duration> },
+ Draining { drain: Drain },
+ Stopped,
+}
+
+pub(crate) struct RebalanceState {
+ phase: Phase,
+ events: Vec<Event>,
+ revokes: u64,
+}
+
+impl Default for RebalanceState {
+ fn default() -> Self {
+ Self {
+ phase: Phase::Starting,
+ events: Vec::new(),
+ revokes: 0,
+ }
+ }
+}
+
+impl RebalanceState {
+ pub(crate) fn on_started(&mut self, drain_timeout: Option<Duration>) {
+ if matches!(self.phase, Phase::Starting) {
+ self.phase = Phase::Consuming { drain_timeout };
+ }
+ }
+
+ pub(crate) fn on_assigned(&mut self, partitions: Vec<TopicPartition>) {
+ self.queue(Event::Assigned(partitions));
+ }
+
+ /// `Some` only before `on_started` or after `on_stopped`, when no loop
+ /// runs to apply the unassign.
+ pub(crate) fn on_revoked(
+ &mut self,
+ partitions: Vec<TopicPartition>,
+ unassign: Unassign,
+ lost: bool,
+ now: Instant,
+ ) -> Option<Unassign> {
+ match self.phase {
+ // A lost assignment belongs to another member, so its commit fails.
+ Phase::Consuming {
+ drain_timeout: Some(timeout),
+ } if !lost => {
+ let revoke = self.next_revoke();
+ self.events.push(Event::Draining {
+ revoke,
+ partitions: partitions.clone(),
+ });
+ self.phase = Phase::Draining {
+ drain: Drain {
+ revoke,
+ partitions,
+ unassign,
+ started: now,
+ timeout,
+ },
+ };
+ None
+ }
+ Phase::Consuming { .. } | Phase::Draining { .. } => {
+ let revoke = self.next_revoke();
+ self.events.push(Event::Revoked {
+ revoke,
+ partitions,
+ unassign,
+ });
+ None
+ }
+ Phase::Starting | Phase::Stopped => Some(unassign),
+ }
+ }
+
+ fn next_revoke(&mut self) -> RevokeId {
+ self.revokes += 1;
+ RevokeId(self.revokes)
+ }
+
+ pub(crate) fn take_events(&mut self) -> Vec<Event> {
+ std::mem::take(&mut self.events)
+ }
+
+ pub(crate) fn drain_deadline(&self) -> Option<Instant> {
+ match &self.phase {
+ Phase::Draining { drain } => Some(drain.started + drain.timeout),
+ _ => None,
+ }
+ }
+
+ pub(crate) fn on_drained(&mut self, revoke: RevokeId) -> Option<Drain> {
+ match &self.phase {
+ Phase::Draining { drain } if drain.revoke == revoke => self.finish_drain(),
+ _ => None,
+ }
+ }
+
+ pub(crate) fn on_drain_timed_out(&mut self, now: Instant) -> Option<Drain> {
+ match self.drain_deadline() {
+ Some(deadline) if now >= deadline => self.finish_drain(),
+ _ => None,
+ }
+ }
+
+ pub(crate) fn on_stopped(&mut self) -> Option<Drain> {
+ match std::mem::replace(&mut self.phase, Phase::Stopped) {
+ Phase::Draining { drain } => Some(drain),
+ _ => None,
+ }
+ }
+
+ pub(crate) fn on_dropped(&mut self) -> Vec<Unassign> {
+ let mut unassigns: Vec<Unassign> = self
+ .on_stopped()
+ .map(|drain| drain.unassign)
+ .into_iter()
+ .collect();
+ unassigns.extend(
+ self.take_events()
+ .into_iter()
+ .filter_map(|event| match event {
+ Event::Revoked { unassign, .. } => Some(unassign),
+ Event::Assigned(_) | Event::Draining { .. } => None,
+ }),
+ );
+ unassigns
+ }
+
+ fn finish_drain(&mut self) -> Option<Drain> {
+ match std::mem::replace(&mut self.phase, Phase::Stopped) {
+ Phase::Draining { drain } => {
+ self.phase = Phase::Consuming {
+ drain_timeout: Some(drain.timeout),
+ };
+ Some(drain)
+ }
+ phase => {
+ self.phase = phase;
+ None
+ }
+ }
+ }
+
+ fn queue(&mut self, event: Event) {
+ if !matches!(self.phase, Phase::Stopped) {
+ self.events.push(event);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const TIMEOUT: Duration = Duration::from_secs(10);
+
+ fn p(partition: i32) -> TopicPartition {
+ TopicPartition::new("events", partition)
+ }
+
+ fn unassign() -> Unassign {
+ Unassign::new(TopicPartitionList::new(), true)
+ }
+
+ fn consuming() -> RebalanceState {
+ let mut state = RebalanceState::default();
+ state.on_started(Some(TIMEOUT));
+ state
+ }
+
+ #[test]
+ fn a_revoke_while_consuming_waits_for_its_drain() {
+ let mut state = consuming();
+ let now = Instant::now();
+
+ assert!(state
+ .on_revoked(vec![p(0)], unassign(), false, now)
+ .is_none());
+ assert_eq!(
+ state.take_events(),
+ vec![Event::Draining {
+ revoke: RevokeId(1),
+ partitions: vec![p(0)],
+ }]
+ );
+ assert_eq!(state.drain_deadline(), Some(now + TIMEOUT));
+ assert!(
+ state.on_drained(RevokeId(2)).is_none(),
+ "another revoke's answer"
+ );
+
+ let drain = state.on_drained(RevokeId(1)).expect("the drain finishes");
+ assert_eq!(drain.partitions, vec![p(0)]);
+ assert_eq!(state.drain_deadline(), None);
+ assert!(
+ state
+ .on_revoked(vec![p(1)], unassign(), false, now)
+ .is_none(),
+ "the next revoke drains too"
+ );
+ }
+
+ fn without_drains() -> RebalanceState {
+ let mut state = RebalanceState::default();
+ state.on_started(None);
+ state
+ }
+
+ fn draining() -> RebalanceState {
+ let mut state = consuming();
+ state.on_revoked(vec![p(9)], unassign(), false, Instant::now());
+ state.take_events();
+ state
+ }
+
+ fn stopped() -> RebalanceState {
+ let mut state = consuming();
+ state.on_stopped();
+ state
+ }
+
+ struct LoopUnassignCase {
+ state: fn() -> RebalanceState,
+ lost: bool,
+ revoke: RevokeId,
+ }
+
+ #[rstest::rstest]
+ #[case::lost(LoopUnassignCase { state: consuming, lost: true, revoke: RevokeId(1) })]
+ #[case::no_drain_timeout(LoopUnassignCase { state: without_drains, lost: false, revoke: RevokeId(1) })]
+ #[case::during_a_drain(LoopUnassignCase { state: draining, lost: false, revoke: RevokeId(2) })]
+ fn a_revoke_that_does_not_drain_goes_to_the_loop_with_its_unassign(
+ #[case] case: LoopUnassignCase,
+ ) {
+ let mut state = (case.state)();
+
+ assert!(state
+ .on_revoked(vec![p(0)], unassign(), case.lost, Instant::now())
+ .is_none());
+ assert_eq!(
+ state.take_events(),
+ vec![Event::Revoked {
+ revoke: case.revoke,
+ partitions: vec![p(0)],
+ unassign: unassign(),
+ }]
+ );
+ }
+
+ #[rstest::rstest]
+ #[case::before_the_loop_starts(RebalanceState::default)]
+ #[case::after_the_loop_stops(stopped)]
+ fn without_a_loop_the_callback_applies_the_unassign(#[case] state: fn() -> RebalanceState) {
+ let mut state = state();
+
+ assert!(state
+ .on_revoked(vec![p(0)], unassign(), false, Instant::now())
+ .is_some());
+ assert_eq!(state.take_events(), vec![]);
+ assert_eq!(state.drain_deadline(), None);
+ }
+
+ #[test]
+ fn a_timed_out_drain_answered_late_does_not_finish_the_next_drain_of_its_partitions() {
+ let mut state = consuming();
+ let now = Instant::now();
+ state.on_revoked(vec![p(0)], unassign(), false, now);
+
+ assert!(state.on_drain_timed_out(now + TIMEOUT / 2).is_none());
+ assert!(state.on_drain_timed_out(now + TIMEOUT).is_some());
+
+ state.on_revoked(vec![p(0)], unassign(), false, now + TIMEOUT);
+ assert!(state.on_drained(RevokeId(1)).is_none());
+ assert!(state.on_drained(RevokeId(2)).is_some());
+ }
+
+ #[test]
+ fn stopping_hands_back_the_drain() {
+ let mut state = draining();
+
+ let drain = state.on_stopped().expect("the drain in progress");
+ assert_eq!(drain.partitions, vec![p(9)]);
+ }
+
+ fn revoked_and_never_taken() -> RebalanceState {
+ let mut state = without_drains();
+ state.on_revoked(vec![p(0)], unassign(), false, Instant::now());
+ state
+ }
+
+ fn draining_then_revoked_and_never_taken() -> RebalanceState {
+ let mut state = draining();
+ state.on_revoked(vec![p(0)], unassign(), false, Instant::now());
+ state
+ }
+
+ struct DroppedCase {
+ state: fn() -> RebalanceState,
+ unassigns: usize,
+ }
+
+ #[rstest::rstest]
+ #[case::nothing_pending(DroppedCase { state: consuming, unassigns: 0 })]
+ #[case::a_revoke_the_loop_never_took(DroppedCase { state: revoked_and_never_taken, unassigns: 1 })]
+ #[case::a_drain_in_progress(DroppedCase { state: draining, unassigns: 1 })]
+ #[case::a_drain_and_a_revoke_the_loop_never_took(DroppedCase { state: draining_then_revoked_and_never_taken, unassigns: 2 })]
+ #[case::after_the_loop_stopped(DroppedCase { state: stopped, unassigns: 0 })]
+ fn a_dropped_loop_hands_back_every_unapplied_unassign(#[case] case: DroppedCase) {
+ let mut state = (case.state)();
+
+ assert_eq!(state.on_dropped().len(), case.unassigns);
+
+ assert_eq!(state.take_events(), vec![]);
+ assert!(
+ state
+ .on_revoked(vec![p(1)], unassign(), false, Instant::now())
+ .is_some(),
+ "stopped: the callback applies the next unassign itself"
+ );
+ }
+}
diff --git a/rust/ingestion-consumer/src/settlement/group.rs b/rust/ingestion-consumer/src/settlement/group.rs
index 4e8a0c7112a5..a69afc9f886a 100644
--- a/rust/ingestion-consumer/src/settlement/group.rs
+++ b/rust/ingestion-consumer/src/settlement/group.rs
@@ -85,6 +85,16 @@ impl<A: Accounting + 'static, D: Admission + 'static> Settlement for GroupSettle
Some(Commit::new(handed_out, resume))
}
+ fn on_assigned(&self, assigned: &[TopicPartition]) {
+ self.forget_partitions(assigned);
+ }
+
+ fn on_revoked(&self, revoked: &[TopicPartition]) {
+ self.forget_partitions(revoked);
+ }
+}
+
+impl<A: Accounting, D: Admission> GroupSettlement<A, D> {
fn forget_partitions(&self, topic_partitions: &[TopicPartition]) {
self.accounting.forget_partitions(topic_partitions);
self.admission.forget_partitions(topic_partitions);
@@ -259,7 +269,7 @@ mod tests {
fn a_revoke_reaches_both_the_accounting_and_the_admission() {
let (settlement, accounting, admission) = settlement();
- settlement.forget_partitions(&[p(0)]);
+ settlement.on_revoked(&[p(0)]);
assert_eq!(accounting.forgotten(), vec![p(0)]);
assert_eq!(admission.forgotten(), vec![p(0)]);
@@ -403,7 +413,7 @@ mod integration_tests {
charge_offsets(&ledger, &p0, 0, &[11]);
settlement.settle_run(&p0, 0, &[Offset(11)]).unwrap();
- settlement.forget_partitions(&[p0]);
+ settlement.on_revoked(&[p0]);
assert!(settlement.take_all().is_none());
}
@@ -506,7 +516,7 @@ mod integration_tests {
assert_eq!(charge(&settlement, &p0(), 10, 10), Some(Pause));
settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();
- settlement.forget_partitions(&[p0()]);
+ settlement.on_revoked(&[p0()]);
assert!(settlement.take_all().is_none());
settlement.settle_run(&p0(), 0, &[Offset(10)]).unwrap();
@@ -515,4 +525,41 @@ mod integration_tests {
"the stale run moves nothing"
);
}
+
+ #[test]
+ fn an_assignment_without_a_revoke_drops_the_partitions_pending_frontier() {
+ let ledger = Arc::new(TopicOffsetLedger::new());
+ let settlement = GroupSettlement::new(
+ Arc::clone(&ledger),
+ Duration::from_secs(3600),
+ BudgetLimits::new(NonZeroU64::new(1), None, 0.5).expect("valid limits"),
+ );
+ let p0 = TopicPartition::new("events", 0);
+ let one_event = [(
+ Offset(10),
+ Charge {
+ events: 1,
+ bytes: 0,
+ },
+ )];
+ assert_eq!(
+ settlement
+ .charge(&p0, 0, &one_event)
+ .map(|charged| charged.pause),
+ Ok(Some(Pause))
+ );
+ settlement.settle_run(&p0, 0, &[Offset(10)]).unwrap();
+
+ settlement.on_assigned(std::slice::from_ref(&p0));
+
+ assert!(settlement.take_all().is_none());
+ assert_eq!(ledger.generation(&p0), 1);
+ assert_eq!(
+ settlement
+ .charge(&p0, 1, &one_event)
+ .map(|charged| charged.pause),
+ Ok(Some(Pause)),
+ "the regained partition starts unpaused"
+ );
+ }
}
diff --git a/rust/ingestion-consumer/src/settlement/mod.rs b/rust/ingestion-consumer/src/settlement/mod.rs
index bb4fc2eea8a7..b74a1390e06e 100644
--- a/rust/ingestion-consumer/src/settlement/mod.rs
+++ b/rust/ingestion-consumer/src/settlement/mod.rs
@@ -38,7 +38,9 @@ pub(crate) trait Settlement: Send + Sync {
fn take_all(&self) -> Option<Commit>;
- fn forget_partitions(&self, topic_partitions: &[TopicPartition]);
+ fn on_assigned(&self, assigned: &[TopicPartition]);
+
+ fn on_revoked(&self, revoked: &[TopicPartition]);
}
pub(crate) trait Accounting: Send + Sync {
diff --git a/rust/ingestion-consumer/tests/batcher_integration_test.rs b/rust/ingestion-consumer/tests/batcher_integration_test.rs
index 5d99506a40d1..1a09d0ad21c0 100644
--- a/rust/ingestion-consumer/tests/batcher_integration_test.rs
+++ b/rust/ingestion-consumer/tests/batcher_integration_test.rs
@@ -8,6 +8,7 @@ use common_kafka_consumer::Partition;
use common::one_message_batcher;
use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;
+use ingestion_consumer::batcher::RevokeId;
use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport};
use ingestion_consumer::routing::RoutingStrategy;
use ingestion_consumer::types::{Accumulator, SerializedKafkaMessage};
@@ -59,7 +60,7 @@ async fn purging_a_just_submitted_batch_is_not_fatal() {
accumulator.push(Partition(0), make_msg("a").into());
batcher.submit(accumulator, &HashMap::new());
// No await, so the task gets the submit and the purge before it runs.
- batcher.revoker().purge_revoked(&[("test".to_string(), 0)]);
+ batcher.revoke(RevokeId(1), &[("test".to_string(), 0)]);
match tokio::time::timeout(Duration::from_millis(100), outputs.errors.recv()).await {
Err(_) => {}
diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rs
index 76b67c5285d4..608c8f064990 100644
--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs
+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs
@@ -2,7 +2,7 @@
mod common;
-use std::collections::HashSet;
+use std::collections::{BTreeSet, HashMap, HashSet};
use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
@@ -590,6 +590,33 @@ struct Harness {
budget: Option<RequestBudget>,
partition_budget: BudgetLimits,
in_flight_limit: ChargeLimit,
+ revoke_drain_timeout: Option<Duration>,
+ rebalance_mode: RebalanceMode,
+}
+
+/// Production runs `group.protocol=consumer` (KIP-848), which also revokes
+/// incrementally. The test broker, Redpanda v25.1.9, does not implement
+/// ConsumerGroupHeartbeat (API key 68) and leaves the key out of its
+/// ApiVersions response. So no case covers KIP-848; `Cooperative` takes the
+/// same `incremental_unassign` path in the consumer.
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum RebalanceMode {
+ Eager,
+ /// `cooperative-sticky`: a revoke takes only the partitions that move.
+ /// The broker is `Stable` while the old owner holds them, and the new owner
+ /// gets them in a second rebalance after the unassign.
+ Cooperative,
+}
+
+impl RebalanceMode {
+ fn configure(self, config: &mut ClientConfig) {
+ match self {
+ Self::Eager => {}
+ Self::Cooperative => {
+ config.set("partition.assignment.strategy", "cooperative-sticky");
+ }
+ }
+ }
}
// Short session and heartbeat so group handovers finish quickly.
@@ -598,7 +625,13 @@ fn make_kafka_consumer(
group_id: &str,
instance_id: Option<&str>,
) -> StreamConsumer<RebalanceContext> {
- make_kafka_consumer_with_context(topic, group_id, instance_id, RebalanceContext::new())
+ make_kafka_consumer_with_context(
+ topic,
+ group_id,
+ instance_id,
+ RebalanceContext::new(),
+ RebalanceMode::Eager,
+ )
}
fn make_kafka_consumer_with_context(
@@ -606,6 +639,7 @@ fn make_kafka_consumer_with_context(
group_id: &str,
instance_id: Option<&str>,
context: RebalanceContext,
+ rebalance_mode: RebalanceMode,
) -> StreamConsumer<RebalanceContext> {
let mut config = ClientConfig::new();
config
@@ -620,6 +654,7 @@ fn make_kafka_consumer_with_context(
if let Some(id) = instance_id {
config.set("group.instance.id", id);
}
+ rebalance_mode.configure(&mut config);
let kafka_consumer: StreamConsumer<RebalanceContext> =
config.create_with_context(context).expect("kafka consumer");
kafka_consumer.subscribe(&[topic]).expect("subscribe");
@@ -678,12 +713,22 @@ impl Harness {
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
)
.await
}
/// One worker; every request carries a 500 ms soft budget.
async fn start_budgeted(topic: &str, partitions: i32) -> Self {
+ Self::start_budgeted_draining_revokes(topic, partitions, None).await
+ }
+
+ async fn start_budgeted_draining_revokes(
+ topic: &str,
+ partitions: i32,
+ revoke_drain_timeout: Option<Duration>,
+ ) -> Self {
Self::start_inner(
topic,
partitions,
@@ -706,6 +751,8 @@ impl Harness {
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ revoke_drain_timeout,
+ RebalanceMode::Eager,
)
.await
}
@@ -731,6 +778,8 @@ impl Harness {
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
)
.await
}
@@ -757,6 +806,34 @@ impl Harness {
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
+ )
+ .await
+ }
+
+ async fn start_draining_revokes(
+ topic: &str,
+ partitions: i32,
+ worker_count: usize,
+ revoke_drain_timeout: Duration,
+ rebalance_mode: RebalanceMode,
+ ) -> Self {
+ Self::start_inner(
+ topic,
+ partitions,
+ worker_count,
+ Duration::from_secs(60),
+ fast_registry_config(),
+ 0,
+ ComponentOptions::new(),
+ common::PRODUCTION_PACK_TARGETS,
+ None,
+ COMMIT_INTERVAL,
+ BudgetLimits::unset(),
+ ChargeLimit::unset(),
+ Some(revoke_drain_timeout),
+ rebalance_mode,
)
.await
}
@@ -775,6 +852,8 @@ impl Harness {
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
)
.await
}
@@ -799,6 +878,8 @@ impl Harness {
commit_interval,
BudgetLimits::unset(),
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
)
.await
}
@@ -842,6 +923,8 @@ impl Harness {
COMMIT_INTERVAL,
partition_budget,
in_flight_limit,
+ None,
+ RebalanceMode::Eager,
)
.await
}
@@ -867,6 +950,8 @@ impl Harness {
commit_interval,
partition_budget,
ChargeLimit::unset(),
+ None,
+ RebalanceMode::Eager,
)
.await
}
@@ -885,6 +970,8 @@ impl Harness {
commit_interval: Duration,
partition_budget: BudgetLimits,
in_flight_limit: ChargeLimit,
+ revoke_drain_timeout: Option<Duration>,
+ rebalance_mode: RebalanceMode,
) -> Self {
create_topic(topic, partitions).await;
@@ -914,8 +1001,13 @@ impl Harness {
let group_id = format!("e2e-{}", Uuid::new_v4());
let ledger = Arc::new(TopicOffsetLedger::new());
let debug_recorder = DebugRecorder::new(10_000, Duration::from_secs(3600));
- let kafka_consumer =
- make_kafka_consumer_with_context(topic, &group_id, None, RebalanceContext::new());
+ let kafka_consumer = make_kafka_consumer_with_context(
+ topic,
+ &group_id,
+ None,
+ RebalanceContext::new(),
+ rebalance_mode,
+ );
let (batcher, outputs) = common::batcher(
pool_source,
@@ -941,6 +1033,7 @@ impl Harness {
ledger: Arc::clone(&ledger),
partition_budget,
in_flight_limit,
+ revoke_drain_timeout,
},
handle,
);
@@ -974,6 +1067,8 @@ impl Harness {
budget,
partition_budget,
in_flight_limit,
+ revoke_drain_timeout,
+ rebalance_mode,
}
}
@@ -989,6 +1084,19 @@ impl Harness {
.collect()
}
+ fn dispatched_assignment_epochs(&self) -> Vec<u64> {
+ self.debug_recorder
+ .backlog()
+ .into_iter()
+ .filter_map(|event| match event.kind {
+ DebugEventKind::BatchDispatched {
+ assignment_epoch, ..
+ } => Some(assignment_epoch),
+ _ => None,
+ })
+ .collect()
+ }
+
/// Abort the consumer task: no shutdown, no drain, no final commit.
fn crash_consumer(&mut self) {
if let Some(task) = self.task.take() {
@@ -1017,6 +1125,7 @@ impl Harness {
&self.group_id,
None,
RebalanceContext::new(),
+ self.rebalance_mode,
);
let (batcher, outputs) = common::batcher(
pool_source,
@@ -1042,6 +1151,7 @@ impl Harness {
ledger: Arc::clone(&self.ledger),
partition_budget: self.partition_budget,
in_flight_limit: self.in_flight_limit,
+ revoke_drain_timeout: self.revoke_drain_timeout,
},
handle,
);
@@ -1113,7 +1223,13 @@ impl Harness {
self.budget,
);
let consumer = IngestionConsumer::from_parts(
- make_kafka_consumer(&self.topic, &self.group_id, None),
+ make_kafka_consumer_with_context(
+ &self.topic,
+ &self.group_id,
+ None,
+ RebalanceContext::new(),
+ self.rebalance_mode,
+ ),
batcher,
outputs,
transport,
@@ -1128,6 +1244,7 @@ impl Harness {
ledger: Arc::new(TopicOffsetLedger::new()),
partition_budget: self.partition_budget,
in_flight_limit: self.in_flight_limit,
+ revoke_drain_timeout: self.revoke_drain_timeout,
},
handle,
);
@@ -1187,6 +1304,97 @@ fn broker_committed_offset(group_id: &str, topic: &str, partition: i32) -> Optio
}
}
+#[derive(Debug)]
+struct GroupState {
+ state: String,
+ owned: Vec<usize>,
+}
+
+impl GroupState {
+ fn handed_over(&self) -> bool {
+ self.state == "Stable" && self.owned == [1, 1]
+ }
+
+ fn holds_a_revoke(&self, mode: RebalanceMode) -> bool {
+ match mode {
+ RebalanceMode::Eager => self.state == "PreparingRebalance" && self.owned.len() == 2,
+ RebalanceMode::Cooperative => self.state == "Stable" && self.owned == [0, 1],
+ }
+ }
+}
+
+fn group_state(group_id: &str) -> Option<GroupState> {
+ let probe: BaseConsumer = ClientConfig::new()
+ .set("bootstrap.servers", KAFKA_BROKERS)
+ .set("group.id", group_id)
+ .create()
+ .expect("group probe");
+ let groups = probe
+ .client()
+ .fetch_group_list(Some(group_id), Duration::from_secs(5))
+ .ok()?;
+ let group = groups
+ .groups()
+ .iter()
+ .find(|group| group.name() == group_id)?;
+ let mut owned: Vec<usize> = group
+ .members()
+ .iter()
+ .map(|member| assigned_partition_count(member.assignment().unwrap_or_default()))
+ .collect();
+ owned.sort_unstable();
+ Some(GroupState {
+ state: group.state().to_string(),
+ owned,
+ })
+}
+
+/// Counts the partitions in a classic-protocol `ConsumerProtocolAssignment`:
+/// an i16 version, then an i32 count of (i16-length topic, i32 count of i32 partitions).
+fn assigned_partition_count(assignment: &[u8]) -> usize {
+ fn read<const N: usize>(bytes: &[u8], at: &mut usize) -> Option<[u8; N]> {
+ let read = bytes.get(*at..*at + N)?.try_into().ok()?;
+ *at += N;
+ Some(read)
+ }
+ fn count(assignment: &[u8]) -> Option<usize> {
+ let mut at = 2;
+ let topics = i32::from_be_bytes(read(assignment, &mut at)?);
+ let mut partitions = 0;
+ for _ in 0..topics {
+ at += i16::from_be_bytes(read(assignment, &mut at)?) as usize;
+ let in_topic = i32::from_be_bytes(read(assignment, &mut at)?) as usize;
+ at += 4 * in_topic;
+ partitions += in_topic;
+ }
+ Some(partitions)
+ }
+ if assignment.is_empty() {
+ return 0;
+ }
+ count(assignment).expect("a well-formed consumer protocol assignment")
+}
+
+async fn wait_for_group(
+ group_id: &str,
+ timeout: Duration,
+ what: &str,
+ cond: impl Fn(&GroupState) -> bool,
+) {
+ let deadline = tokio::time::Instant::now() + timeout;
+ loop {
+ let observed = group_state(group_id);
+ if observed.as_ref().is_some_and(&cond) {
+ return;
+ }
+ assert!(
+ tokio::time::Instant::now() < deadline,
+ "timed out waiting for the group to {what}, last seen {observed:?}"
+ );
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ }
+}
+
async fn wait_until(timeout: Duration, msg: &str, mut cond: impl FnMut() -> bool) {
let deadline = tokio::time::Instant::now() + timeout;
while !cond() {
@@ -1459,6 +1667,100 @@ async fn a_partial_ack_after_a_revoke_resends_nothing() {
harness.stop().await;
}
+#[tokio::test]
+async fn a_partial_ack_during_a_drain_commits_only_what_was_processed() {
+ let topic = format!("e2e-partial-drain-{}", Uuid::new_v4());
+ let harness =
+ Harness::start_budgeted_draining_revokes(&topic, 1, Some(Duration::from_secs(20))).await;
+ // The partition holds a, b, a, b, and the b messages come back unfinished.
+ *harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-b".to_string(), 1));
+ let records = alternating_visitors(0, "tok:203.0.113.7", 0);
+
+ // A trickle keeps the consumer polling, so it takes part in the rebalance.
+ let producer = make_producer();
+ producer
+ .client()
+ .fetch_metadata(Some(&topic), Duration::from_secs(5))
+ .expect("trickle producer metadata");
+
+ let gate = harness.workers[0].block().await;
+ produce_keyed_in_one_transaction(&topic, "tok", &records).await;
+ let trickling = Arc::new(AtomicBool::new(true));
+ let trickle = tokio::spawn({
+ let trickling = Arc::clone(&trickling);
+ let topic = topic.clone();
+ async move {
+ let mut seq = 1000usize;
+ while trickling.load(Ordering::SeqCst) {
+ produce(&producer, &topic, 0, "tok", "trickle", seq).await;
+ seq += 1;
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ }
+ }
+ });
+ wait_until(
+ Duration::from_secs(15),
+ "a request to reach the worker",
+ || harness.workers[0].arrived_count() > 0,
+ )
+ .await;
+
+ let second = harness.join_second_consumer().await;
+ // The drain waits on the gated request.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ trickling.store(false, Ordering::SeqCst);
+ trickle.await.expect("trickle task");
+ drop(gate);
+
+ wait_until(
+ Duration::from_secs(30),
+ "every message to be delivered",
+ || {
+ let delivered: HashSet<(String, usize)> = harness
+ .delivery_log
+ .lock()
+ .unwrap()
+ .iter()
+ .cloned()
+ .collect();
+ records
+ .iter()
+ .all(|(_, _, visitor, seq)| delivered.contains(&(visitor.to_string(), *seq)))
+ },
+ )
+ .await;
+ // Time for a replay by the new owner to arrive.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ assert!(
+ harness.workers[0]
+ .leave_unfinished
+ .lock()
+ .unwrap()
+ .is_none(),
+ "the gated request was acked as partial"
+ );
+ assert_eq!(
+ *harness.workers[0].replayed.lock().unwrap(),
+ Vec::<(String, usize)>::new(),
+ "the old owner resends nothing of a draining partition"
+ );
+ let log = harness.delivery_log.lock().unwrap().clone();
+ let mut seen = HashSet::new();
+ let processed_twice: BTreeSet<(String, usize)> = log
+ .iter()
+ .filter(|entry| entry.0 != "trickle" && !seen.insert(*entry))
+ .cloned()
+ .collect();
+ assert_eq!(
+ processed_twice,
+ BTreeSet::from([("visitor-a".to_string(), 2)])
+ );
+
+ second.stop().await;
+ harness.stop().await;
+}
+
#[tokio::test]
async fn a_message_that_never_fits_the_budget_lands_without_one() {
let topic = format!("e2e-partial-escalate-{}", Uuid::new_v4());
@@ -3370,6 +3672,321 @@ async fn second_consumer_joining_the_group_preserves_all_messages() {
harness.stop().await;
}
+#[rstest]
+#[case::eager(RebalanceMode::Eager)]
+#[case::cooperative(RebalanceMode::Cooperative)]
+#[tokio::test]
+async fn a_drain_commits_the_in_flight_messages_before_the_new_owner_reads(
+ #[case] rebalance_mode: RebalanceMode,
+) {
+ let topic = format!("e2e-revoke-drain-{}", Uuid::new_v4());
+ let harness =
+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)
+ .await;
+ let producer = make_producer();
+
+ let mut guards = Vec::new();
+ for w in &harness.workers {
+ guards.push(w.block().await);
+ }
+ for seq in 0..5usize {
+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;
+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;
+ }
+ wait_until(Duration::from_secs(10), "work to reach a worker", || {
+ harness.workers.iter().any(|w| w.arrived_count() > 0)
+ })
+ .await;
+
+ let second = harness.join_second_consumer().await;
+ // The drain waits on the gated requests.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ guards.clear();
+ for seq in 5..10usize {
+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;
+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;
+ }
+
+ let expected: HashSet<(String, usize)> = ["user-1", "user-2"]
+ .iter()
+ .flat_map(|user| (0..10usize).map(|seq| (user.to_string(), seq)))
+ .collect();
+ wait_until(
+ Duration::from_secs(30),
+ "every message to be delivered",
+ || {
+ let delivered: HashSet<(String, usize)> = harness
+ .delivery_log
+ .lock()
+ .unwrap()
+ .iter()
+ .cloned()
+ .collect();
+ expected.is_subset(&delivered)
+ },
+ )
+ .await;
+ // Time for a replay by the new owner to arrive.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ let log = harness.delivery_log.lock().unwrap().clone();
+ let mut seen = HashSet::new();
+ let duplicates: Vec<&(String, usize)> =
+ log.iter().filter(|entry| !seen.insert(*entry)).collect();
+ assert!(
+ duplicates.is_empty(),
+ "the new owner processed again what the old owner finished: {duplicates:?}"
+ );
+
+ second.stop().await;
+ harness.stop().await;
+}
+
+fn processed_counts(harness: &Harness) -> HashMap<(String, usize), usize> {
+ let mut counts = HashMap::new();
+ for entry in harness.delivery_log.lock().unwrap().iter() {
+ *counts.entry(entry.clone()).or_insert(0) += 1;
+ }
+ counts
+}
+
+#[rstest]
+#[case::eager(RebalanceMode::Eager)]
+#[case::cooperative(RebalanceMode::Cooperative)]
+#[tokio::test]
+async fn a_drain_holds_every_partition_and_dispatches_nothing_until_its_release(
+ #[case] rebalance_mode: RebalanceMode,
+) {
+ let topic = format!("e2e-drain-hold-{}", Uuid::new_v4());
+ let harness =
+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)
+ .await;
+ let producer = make_producer();
+
+ let mut guards = Vec::new();
+ for w in &harness.workers {
+ guards.push(w.block().await);
+ }
+ for seq in 0..5usize {
+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;
+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;
+ }
+ wait_until(
+ Duration::from_secs(10),
+ "every produced message to dispatch",
+ || harness.poll_sizes().iter().sum::<usize>() == 10,
+ )
+ .await;
+
+ let second = harness.join_second_consumer().await;
+ wait_for_group(
+ &harness.group_id,
+ Duration::from_secs(10),
+ "hold a revoke",
+ |group| group.holds_a_revoke(rebalance_mode),
+ )
+ .await;
+ // The revoke arrives on the next heartbeat; the gated requests hold the drain.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ let dispatched = harness.poll_sizes().len();
+ for seq in 5..15usize {
+ produce(&producer, &topic, 0, "tok", "user-1", seq).await;
+ produce(&producer, &topic, 1, "tok", "user-2", seq).await;
+ }
+ tokio::time::sleep(Duration::from_secs(2)).await;
+ assert_eq!(
+ harness.poll_sizes().len(),
+ dispatched,
+ "librdkafka pauses the whole assignment until the unassign, so a drain dispatches no poll"
+ );
+ let group = group_state(&harness.group_id);
+ assert!(
+ group
+ .as_ref()
+ .is_some_and(|group| group.holds_a_revoke(rebalance_mode)),
+ "the drain still holds the revoke: {group:?}"
+ );
+
+ guards.clear();
+ let expected: HashSet<(String, usize)> = ["user-1", "user-2"]
+ .iter()
+ .flat_map(|user| (0..15usize).map(|seq| (user.to_string(), seq)))
+ .collect();
+ wait_until(
+ Duration::from_secs(30),
+ "every message to be delivered",
+ || {
+ let delivered: HashSet<(String, usize)> = harness
+ .delivery_log
+ .lock()
+ .unwrap()
+ .iter()
+ .cloned()
+ .collect();
+ expected.is_subset(&delivered)
+ },
+ )
+ .await;
+ wait_for_group(
+ &harness.group_id,
+ Duration::from_secs(10),
+ "hand the partition over",
+ GroupState::handed_over,
+ )
+ .await;
+ // Time for a replay by either owner to arrive.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ let duplicates: Vec<((String, usize), usize)> = processed_counts(&harness)
+ .into_iter()
+ .filter(|(_, count)| *count > 1)
+ .collect();
+ assert!(
+ duplicates.is_empty(),
+ "processed more than once: {duplicates:?}"
+ );
+
+ second.stop().await;
+ harness.stop().await;
+}
+
+#[rstest]
+#[case::eager(RebalanceMode::Eager)]
+#[case::cooperative(RebalanceMode::Cooperative)]
+#[tokio::test]
+async fn a_drain_that_times_out_hands_the_partitions_over_uncommitted(
+ #[case] rebalance_mode: RebalanceMode,
+) {
+ let topic = format!("e2e-drain-timeout-{}", Uuid::new_v4());
+ let harness =
+ Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(3), rebalance_mode).await;
+ let producer = make_producer();
+ let partitions = [
+ TopicPartition::new(topic.clone(), 0),
+ TopicPartition::new(topic.clone(), 1),
+ ];
+ let users = [("user-1", 0), ("user-2", 1)];
+
+ let mut guards = Vec::new();
+ for w in &harness.workers {
+ guards.push(w.block().await);
+ }
+ for seq in 0..5usize {
+ for (user, partition) in users {
+ produce(&producer, &topic, partition, "tok", user, seq).await;
+ }
+ }
+ wait_until(
+ Duration::from_secs(10),
+ "every produced message to dispatch",
+ || harness.poll_sizes().iter().sum::<usize>() == 10,
+ )
+ .await;
+ let generations_before: Vec<u64> = partitions
+ .iter()
+ .map(|partition| harness.ledger.generation(partition))
+ .collect();
+ let bumps = |harness: &Harness| -> Vec<u64> {
+ let mut bumps: Vec<u64> = partitions
+ .iter()
+ .zip(&generations_before)
+ .map(|(partition, before)| harness.ledger.generation(partition) - before)
+ .collect();
+ bumps.sort_unstable();
+ bumps
+ };
+
+ let joined = tokio::time::Instant::now();
+ let second = harness.join_second_consumer().await;
+ // The gated requests outlive the drain timeout.
+ wait_for_group(
+ &harness.group_id,
+ Duration::from_secs(20),
+ "hand the partition over",
+ GroupState::handed_over,
+ )
+ .await;
+ assert!(
+ joined.elapsed() >= Duration::from_secs(3),
+ "the drain held the revoke for its timeout"
+ );
+ assert_eq!(harness.committed_offset(0), None);
+ assert_eq!(harness.committed_offset(1), None);
+ // The release bumps the generation of each revoked partition, and an
+ // assignment bumps the generation of each partition it returns.
+ let expected_bumps = match rebalance_mode {
+ RebalanceMode::Eager => vec![1, 2],
+ RebalanceMode::Cooperative => vec![0, 1],
+ };
+ wait_until(
+ Duration::from_secs(10),
+ "the first consumer to apply its new assignment",
+ || bumps(&harness) == expected_bumps,
+ )
+ .await;
+ let moved = partitions
+ .iter()
+ .zip(&generations_before)
+ .find(|(partition, before)| harness.ledger.generation(partition) == *before + 1)
+ .map(|(partition, _)| partition.partition)
+ .expect("one partition left the first consumer");
+
+ guards.clear();
+ let read_again = |partition: i32| partition == moved || rebalance_mode == RebalanceMode::Eager;
+ let processed_as_expected = |harness: &Harness| {
+ let counts = processed_counts(harness);
+ users.iter().all(|(user, partition)| {
+ let expected = if read_again(*partition) { 2 } else { 1 };
+ (0..5usize).all(|seq| counts.get(&(user.to_string(), seq)) == Some(&expected))
+ })
+ };
+ wait_until(
+ Duration::from_secs(30),
+ "the released requests and the new owners to process every message",
+ || processed_as_expected(&harness),
+ )
+ .await;
+ wait_until(
+ Duration::from_secs(10),
+ "each owner to commit its partition",
+ || harness.committed_offset(0) == Some(5) && harness.committed_offset(1) == Some(5),
+ )
+ .await;
+ // Time for a replay by either owner to arrive.
+ tokio::time::sleep(Duration::from_secs(2)).await;
+
+ assert!(
+ processed_as_expected(&harness),
+ "the old owner's released requests process every message once, and each new assignment processes its partition once more: {:?}",
+ processed_counts(&harness)
+ );
+ assert_eq!(
+ bumps(&harness),
+ expected_bumps,
+ "no later rebalance touched the first consumer's partitions"
+ );
+ let commits_of_moved: Vec<i64> = harness
+ .debug_recorder
+ .backlog()
+ .into_iter()
+ .filter_map(|event| match event.kind {
+ DebugEventKind::CommitSubmitted { offsets } => Some(offsets),
+ _ => None,
+ })
+ .flatten()
+ .filter(|offset| offset.partition == moved)
+ .map(|offset| offset.offset)
+ .collect();
+ assert_eq!(
+ commits_of_moved,
+ Vec::<i64>::new(),
+ "the old owner's late completions commit nothing for the partition it handed over"
+ );
+
+ second.stop().await;
+ harness.stop().await;
+}
+
#[tokio::test]
async fn partition_lost_and_regained_keeps_the_consumer_alive() {
let topic = format!("e2e-regain-{}", Uuid::new_v4());
@@ -3471,6 +4088,11 @@ async fn partition_lost_and_regained_keeps_the_consumer_alive() {
},
)
.await;
+ let epochs = harness.dispatched_assignment_epochs();
+ assert!(
+ epochs.last() > epochs.first(),
+ "polls after the regain go out under a later assignment epoch: {epochs:?}"
+ );
harness.stop().await;
}
@@ -3516,6 +4138,7 @@ async fn fenced_static_member_exits_on_fatal_error() {
ledger: Arc::new(TopicOffsetLedger::new()),
partition_budget: BudgetLimits::unset(),
in_flight_limit: ChargeLimit::unset(),
+ revoke_drain_timeout: None,
},
handle,
);
diff --git a/rust/ingestion-consumer/tests/grpc_transport_test.rs b/rust/ingestion-consumer/tests/grpc_transport_test.rs
index ef9f4092a13f..79f980f56cf8 100644
--- a/rust/ingestion-consumer/tests/grpc_transport_test.rs
+++ b/rust/ingestion-consumer/tests/grpc_transport_test.rs
@@ -9,6 +9,7 @@ use std::time::Duration;
use common::one_message_batcher;
use common_kafka_consumer::Partition;
use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;
+use ingestion_consumer::batcher::BatcherCompletion;
use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport, SendOptions};
use ingestion_consumer::routing::RoutingStrategy;
use ingestion_consumer::transport::{SendOutcome, TransportError};
@@ -1396,6 +1397,9 @@ async fn a_busy_send_is_retried_until_it_completes() {
.await
.expect("successful retry completes")
.expect("completion channel stays open");
+ let BatcherCompletion::Group(completion) = completion else {
+ panic!("expected a group completion");
+ };
assert_eq!(completion.accepted, 1);
tokio::time::sleep(Duration::from_millis(550)).await;
assert!(
@@ -1440,6 +1444,9 @@ async fn the_watchdog_allows_in_flight_success_after_the_deadline() {
.await
.expect("late successful send completes")
.expect("completion channel stays open");
+ let BatcherCompletion::Group(completion) = completion else {
+ panic!("expected a group completion");
+ };
assert_eq!(completion.accepted, 1);
tokio::time::sleep(Duration::from_millis(150)).await;