:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Requires Kafka on localhost:9092 (docker-compose).
mod common;
use std::collections::{BTreeSet, HashMap, HashSet};
use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
use axum::routing::get;
use axum::Router;
use futures::StreamExt;
use lifecycle::{ComponentOptions, Manager};
use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
use rdkafka::config::ClientConfig;
use rdkafka::consumer::{BaseConsumer, Consumer, StreamConsumer};
use rdkafka::message::{Header, OwnedHeaders};
use rdkafka::producer::{BaseProducer, BaseRecord, FutureProducer, FutureRecord, Producer};
use rdkafka::util::Timeout;
use rdkafka::TopicPartitionList;
use rstest::rstest;
use tokio::net::TcpListener;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
use common_kafka_consumer::{TopicOffsetLedger, TopicPartition};
use ingestion_consumer::batcher::packer::PackTargets;
use ingestion_consumer::batcher::state_machine::RequestBudget;
use ingestion_consumer::batcher::worker_pool::WorkerPoolSource;
use ingestion_consumer::batcher::BatcherObserver;
use ingestion_consumer::charge_limit::ChargeLimit;
use ingestion_consumer::consumer::{IngestionConsumer, IngestionConsumerOptions};
use ingestion_consumer::debug_recorder::{DebugEventKind, DebugRecorder};
use ingestion_consumer::discovery::reconcile_membership;
use ingestion_consumer::grpc_transport::{GrpcPort, GrpcTransport};
use ingestion_consumer::partition_budget::BudgetLimits;
use ingestion_consumer::rebalance::RebalanceContext;
use ingestion_consumer::routing::RoutingStrategy;
use ingestion_consumer::worker_registry::{
WorkerId, WorkerRegistry, WorkerRegistryConfig, WorkerState,
};
use ingestion_worker_proto::ingestion::worker::v1::worker_ingest_server::{
WorkerIngest as WorkerIngestService, WorkerIngestServer,
};
use ingestion_worker_proto::ingestion::worker::v1::{
ingest_stream_request, ingest_stream_response, IngestStreamRequest, IngestStreamResponse,
MessageRef, StreamReady, SubBatchAck, SubBatchStatus,
};
const UNBOUNDED_BYTES: u64 = 1 << 40;
const KAFKA_BROKERS: &str = "localhost:9092";
const BUDGET_ATTEMPTS: NonZeroU32 = NonZeroU32::new(3).expect("nonzero attempts");
async fn create_topic(name: &str, partitions: i32) {
let admin: AdminClient<DefaultClientContext> = ClientConfig::new()
.set("bootstrap.servers", KAFKA_BROKERS)
.create()
.expect("admin client");
let topic = NewTopic::new(name, partitions, TopicReplication::Fixed(1));
let opts = AdminOptions::new().operation_timeout(Some(Duration::from_secs(10)));
for result in admin.create_topics(&[topic], &opts).await.unwrap() {
match result {
Ok(_) | Err((_, rdkafka::types::RDKafkaErrorCode::TopicAlreadyExists)) => {}
Err((name, err)) => panic!("failed to create topic {name}: {err:?}"),
}
}
tokio::time::sleep(Duration::from_millis(300)).await;
}
fn make_producer() -> FutureProducer {
ClientConfig::new()
.set("bootstrap.servers", KAFKA_BROKERS)
.set("message.timeout.ms", "5000")
.create()
.expect("producer")
}
async fn produce(
producer: &FutureProducer,
topic: &str,
partition: i32,
token: &str,
distinct_id: &str,
seq: usize,
) {
let value = format!(r#"{{"seq":{seq}}}"#);
let key = format!("{token}:{distinct_id}");
let headers = OwnedHeaders::new()
.insert(Header {
key: "token",
value: Some(token),
})
.insert(Header {
key: "distinct_id",
value: Some(distinct_id),
});
producer
.send(
FutureRecord::to(topic)
.key(&key)
.payload(&value)
.partition(partition)
.headers(headers),
Timeout::After(Duration::from_secs(5)),
)
.await
.expect("produce failed");
}
async fn produce_in_one_transaction(
topic: &str,
partition: i32,
token: &str,
records: &[(&str, usize)],
) {
let records = records
.iter()
.map(|&(distinct_id, seq)| (format!("{token}:{distinct_id}"), distinct_id, seq))
.collect::<Vec<_>>();
let records: Vec<(i32, &str, &str, usize)> = records
.iter()
.map(|(key, distinct_id, seq)| (partition, key.as_str(), *distinct_id, *seq))
.collect();
produce_keyed_in_one_transaction(topic, token, &records).await;
}
fn alternating_visitors(
partition: i32,
key: &str,
first_seq: usize,
) -> [(i32, &str, &str, usize); 4] {
[
(partition, key, "visitor-a", first_seq),
(partition, key, "visitor-b", first_seq + 1),
(partition, key, "visitor-a", first_seq + 2),
(partition, key, "visitor-b", first_seq + 3),
]
}
async fn produce_keyed_in_one_transaction(
topic: &str,
token: &str,
records: &[(i32, &str, &str, usize)],
) {
let topic = topic.to_string();
let token = token.to_string();
let records: Vec<(i32, String, String, usize)> = records
.iter()
.map(|&(partition, key, distinct_id, seq)| {
(partition, key.to_string(), distinct_id.to_string(), seq)
})
.collect();
// The transaction calls block, so keep them off the test's runtime thread,
// which also serves the fake workers.
tokio::task::spawn_blocking(move || {
let producer: BaseProducer = ClientConfig::new()
.set("bootstrap.servers", KAFKA_BROKERS)
.set("message.timeout.ms", "5000")
.set("transactional.id", format!("e2e-{}", Uuid::new_v4()))
.create()
.expect("producer");
let timeout = Timeout::After(Duration::from_secs(10));
producer
.init_transactions(timeout)
.expect("init transactions");
producer.begin_transaction().expect("begin transaction");
for (partition, key, distinct_id, seq) in &records {
let value = format!(r#"{{"seq":{seq}}}"#);
let headers = OwnedHeaders::new()
.insert(Header {
key: "token",
value: Some(token.as_str()),
})
.insert(Header {
key: "distinct_id",
value: Some(distinct_id.as_str()),
});
producer
.send(
BaseRecord::to(&topic)
.key(key.as_str())
.payload(&value)
.partition(*partition)
.headers(headers),
)
.map_err(|(err, _)| err)
.expect("enqueue failed");
}
producer
.commit_transaction(timeout)
.expect("commit transaction");
})
.await
.expect("transactional produce panicked");
}
async fn produce_raw(
producer: &FutureProducer,
topic: &str,
partition: i32,
key: &str,
payload: &[u8],
headers: Option<OwnedHeaders>,
) {
let mut record = FutureRecord::to(topic)
.key(key)
.payload(payload)
.partition(partition);
if let Some(h) = headers {
record = record.headers(h);
}
producer
.send(record, Timeout::After(Duration::from_secs(5)))
.await
.expect("produce failed");
}
struct FakeWorker {
pub url: String,
pub received: Arc<Mutex<Vec<(String, usize)>>>,
/// Message count of each recorded sub-batch, in arrival order.
pub batch_sizes: Arc<Mutex<Vec<usize>>>,
/// When false, /_ready fails and every send nacks.
pub healthy: Arc<AtomicBool>,
/// When false, the worker stays ready but nacks every send.
pub ingest_ok: Arc<AtomicBool>,
/// When set, the next sub-batch is recorded, then nacked, so its replay duplicates it.
pub ack_lost_once: Arc<AtomicBool>,
/// When true, every sub-batch is nacked unrecorded.
pub reject_4xx: Arc<AtomicBool>,
/// When set to `(distinct_id, frames)`, the next `frames` sub-batches with
/// a soft budget that carry this distinct_id leave all of that id's
/// messages unfinished. They are not recorded, and the ack is PARTIAL.
pub leave_unfinished: Arc<Mutex<Option<(String, usize)>>>,
/// (distinct_id, seq) pairs that arrived in a sub-batch marked as replay.
pub replayed: Arc<Mutex<Vec<(String, usize)>>>,
/// When true, replay sub-batches get no answer and stay in flight.
pub hold_replays: Arc<AtomicBool>,
/// When set, the next sub-batch is recorded in full but acked as one message fewer.
pub underreport_once: Arc<AtomicBool>,
arrived: Arc<AtomicUsize>,
on_arrival: ArrivalHook,
gate: Arc<tokio::sync::Mutex<()>>,
_task: tokio::task::JoinHandle<()>,
}
/// Shared by all workers, so its mutex puts every accepted message in one total order.
type DeliveryLog = Arc<Mutex<Vec<(String, usize)>>>;
type ArrivalHook = Arc<Mutex<Option<Box<dyn Fn() + Send + Sync>>>>;
impl FakeWorker {
async fn start() -> Self {
Self::start_inner(None).await
}
async fn start_logged(delivery_log: DeliveryLog) -> Self {
Self::start_inner(Some(delivery_log)).await
}
async fn start_inner(delivery_log: Option<DeliveryLog>) -> Self {
let received: Arc<Mutex<Vec<(String, usize)>>> = Arc::new(Mutex::new(Vec::new()));
let batch_sizes: Arc<Mutex<Vec<usize>>> = Arc::new(Mutex::new(Vec::new()));
let healthy = Arc::new(AtomicBool::new(true));
let ingest_ok = Arc::new(AtomicBool::new(true));
let ack_lost_once = Arc::new(AtomicBool::new(false));
let reject_4xx = Arc::new(AtomicBool::new(false));
let underreport_once = Arc::new(AtomicBool::new(false));
let leave_unfinished = Arc::new(Mutex::new(None));
let replayed = Arc::new(Mutex::new(Vec::new()));
let hold_replays = Arc::new(AtomicBool::new(false));
let arrived = Arc::new(AtomicUsize::new(0));
let on_arrival: ArrivalHook = Arc::new(Mutex::new(None));
let gate = Arc::new(tokio::sync::Mutex::new(()));
let app = Router::new().route(
"/_ready",
get({
let h = Arc::clone(&healthy);
move || {
let h = h.clone();
async move {
if h.load(Ordering::Relaxed) {
axum::http::StatusCode::OK
} else {
axum::http::StatusCode::SERVICE_UNAVAILABLE
}
}
}
}),
);
// gRPC listens on the HTTP port + 1, as `GrpcPort::OffsetFromHttp(1)` expects.
let (listener, grpc_listener) = loop {
let http = TcpListener::bind("127.0.0.1:0").await.unwrap();
let Some(grpc_port) = http.local_addr().unwrap().port().checked_add(1) else {
continue;
};
match TcpListener::bind(("127.0.0.1", grpc_port)).await {
Ok(grpc) => break (http, grpc),
Err(_) => continue,
}
};
let addr = listener.local_addr().unwrap();
let url = format!("http://127.0.0.1:{}", addr.port());
let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let grpc_service = FakeWorkerGrpc {
received: Arc::clone(&received),
batch_sizes: Arc::clone(&batch_sizes),
healthy: Arc::clone(&healthy),
ingest_ok: Arc::clone(&ingest_ok),
ack_lost_once: Arc::clone(&ack_lost_once),
reject_4xx: Arc::clone(&reject_4xx),
underreport_once: Arc::clone(&underreport_once),
leave_unfinished: Arc::clone(&leave_unfinished),
replayed: Arc::clone(&replayed),
hold_replays: Arc::clone(&hold_replays),
arrived: Arc::clone(&arrived),
on_arrival: Arc::clone(&on_arrival),
gate: Arc::clone(&gate),
delivery_log,
};
tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(
WorkerIngestServer::new(grpc_service)
.accept_compressed(tonic::codec::CompressionEncoding::Gzip)
.send_compressed(tonic::codec::CompressionEncoding::Gzip),
)
.serve_with_incoming(tokio_stream::wrappers::TcpListenerStream::new(
grpc_listener,
))
.await
.unwrap();
});
Self {
url,
received,
batch_sizes,
healthy,
ingest_ok,
ack_lost_once,
reject_4xx,
underreport_once,
leave_unfinished,
replayed,
hold_replays,
arrived,
on_arrival,
gate,
_task: task,
}
}
fn count(&self) -> usize {
self.received.lock().unwrap().len()
}
fn batch_sizes(&self) -> Vec<usize> {
self.batch_sizes.lock().unwrap().clone()
}
/// Sub-batches that have reached this worker (including any held by the gate).
fn arrived_count(&self) -> usize {
self.arrived.load(Ordering::SeqCst)
}
/// Freeze this worker's ingest handling: sub-batches arrive but block until the
/// returned guard is dropped. Acquire before producing so the batch is caught.
async fn block(&self) -> tokio::sync::OwnedMutexGuard<()> {
Arc::clone(&self.gate).lock_owned().await
}
/// Run `hook` as each sub-batch arrives, before the gate holds it.
fn on_arrival(&self, hook: impl Fn() + Send + Sync + 'static) {
*self.on_arrival.lock().unwrap() = Some(Box::new(hook));
}
fn seqs_for(&self, distinct_id: &str) -> Vec<usize> {
self.received
.lock()
.unwrap()
.iter()
.filter(|(did, _)| did == distinct_id)
.map(|(_, seq)| *seq)
.collect()
}
}
// A nack ends the stream, as the protocol requires.
struct FakeWorkerGrpc {
received: Arc<Mutex<Vec<(String, usize)>>>,
batch_sizes: Arc<Mutex<Vec<usize>>>,
healthy: Arc<AtomicBool>,
ingest_ok: Arc<AtomicBool>,
ack_lost_once: Arc<AtomicBool>,
reject_4xx: Arc<AtomicBool>,
underreport_once: Arc<AtomicBool>,
leave_unfinished: Arc<Mutex<Option<(String, usize)>>>,
replayed: Arc<Mutex<Vec<(String, usize)>>>,
hold_replays: Arc<AtomicBool>,
arrived: Arc<AtomicUsize>,
on_arrival: ArrivalHook,
gate: Arc<tokio::sync::Mutex<()>>,
delivery_log: Option<DeliveryLog>,
}
#[tonic::async_trait]
impl WorkerIngestService for FakeWorkerGrpc {
type IngestStreamStream = tokio_stream::wrappers::UnboundedReceiverStream<
Result<IngestStreamResponse, tonic::Status>,
>;
async fn ingest_stream(
&self,
request: tonic::Request<tonic::Streaming<IngestStreamRequest>>,
) -> Result<tonic::Response<Self::IngestStreamStream>, tonic::Status> {
let mut inbound = request.into_inner();
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let received = Arc::clone(&self.received);
let batch_sizes = Arc::clone(&self.batch_sizes);
let healthy = Arc::clone(&self.healthy);
let ingest_ok = Arc::clone(&self.ingest_ok);
let ack_lost_once = Arc::clone(&self.ack_lost_once);
let reject_4xx = Arc::clone(&self.reject_4xx);
let underreport_once = Arc::clone(&self.underreport_once);
let leave_unfinished = Arc::clone(&self.leave_unfinished);
let replayed = Arc::clone(&self.replayed);
let hold_replays = Arc::clone(&self.hold_replays);
let arrived = Arc::clone(&self.arrived);
let on_arrival = Arc::clone(&self.on_arrival);
let gate = Arc::clone(&self.gate);
let delivery_log = self.delivery_log.clone();
tokio::spawn(async move {
// Mirror the real worker: greet with `ready` so response headers
// flush; the worker stream must ignore it.
let _ = tx.send(Ok(IngestStreamResponse {
msg: Some(ingest_stream_response::Msg::Ready(StreamReady {})),
}));
let nack = |tx: &tokio::sync::mpsc::UnboundedSender<
Result<IngestStreamResponse, tonic::Status>,
>,
seq: u64,
reason: &str| {
let _ = tx.send(Ok(IngestStreamResponse {
msg: Some(ingest_stream_response::Msg::Ack(SubBatchAck {
seq,
status: SubBatchStatus::Failed as i32,
accepted: 0,
error: reason.to_string(),
unfinished: vec![],
})),
}));
};
while let Ok(Some(frame)) = inbound.message().await {
let Some(ingest_stream_request::Msg::SubBatch(sub_batch)) = frame.msg else {
continue;
};
arrived.fetch_add(1, Ordering::SeqCst);
if let Some(hook) = on_arrival.lock().unwrap().as_ref() {
hook();
}
let _hold = gate.lock().await;
if !healthy.load(Ordering::Relaxed) || !ingest_ok.load(Ordering::Relaxed) {
nack(&tx, sub_batch.seq, "worker unhealthy");
return;
}
if reject_4xx.load(Ordering::SeqCst) {
nack(&tx, sub_batch.seq, "poison batch");
return;
}
if sub_batch.replay && hold_replays.load(Ordering::SeqCst) {
continue;
}
let entries: Vec<(String, usize)> = sub_batch
.messages
.iter()
.map(|msg| {
let did = msg.headers.get("distinct_id").cloned().unwrap_or_default();
let seq = msg
.value
.as_deref()
.and_then(|v| serde_json::from_str::<serde_json::Value>(v).ok())
.and_then(|v| v["seq"].as_u64())
.unwrap_or(0) as usize;
(did, seq)
})
.collect();
let unfinished_id = if sub_batch.soft_budget_ms > 0 {
leave_unfinished
.lock()
.unwrap()
.as_ref()
.map(|(id, _)| id.clone())
} else {
None
};
let mut done = Vec::new();
let mut unfinished = Vec::new();
for (message, entry) in sub_batch.messages.iter().zip(entries) {
if unfinished_id.as_deref() == Some(entry.0.as_str()) {
unfinished.push(MessageRef {
partition: message.partition,
offset: message.offset,
});
} else {
done.push(entry);
}
}
if sub_batch.replay {
replayed.lock().unwrap().extend(done.iter().cloned());
}
received.lock().unwrap().extend(done.iter().cloned());
batch_sizes.lock().unwrap().push(sub_batch.messages.len());
if let Some(log) = &delivery_log {
log.lock().unwrap().extend(done.iter().cloned());
}
if !unfinished.is_empty() {
let mut frames_left = leave_unfinished.lock().unwrap();
let used_up = frames_left.as_mut().is_some_and(|(_, frames)| {
*frames -= 1;
*frames == 0
});
if used_up {
*frames_left = None;
}
drop(frames_left);
let _ = tx.send(Ok(IngestStreamResponse {
msg: Some(ingest_stream_response::Msg::Ack(SubBatchAck {
seq: sub_batch.seq,
status: SubBatchStatus::Partial as i32,
accepted: done.len() as u32,
error: String::new(),
unfinished,
})),
}));
continue;
}
if ack_lost_once.swap(false, Ordering::SeqCst) {
nack(&tx, sub_batch.seq, "ack lost");
return;
}
let accepted = sub_batch.messages.len() as u32;
let reported = if underreport_once.swap(false, Ordering::SeqCst) {
accepted.saturating_sub(1)
} else {
accepted
};
let _ = tx.send(Ok(IngestStreamResponse {
msg: Some(ingest_stream_response::Msg::Ack(SubBatchAck {
seq: sub_batch.seq,
status: SubBatchStatus::Ok as i32,
accepted: reported,
error: String::new(),
unfinished: vec![],
})),
}));
}
});
Ok(tonic::Response::new(
tokio_stream::wrappers::UnboundedReceiverStream::new(rx),
))
}
}
const COMMIT_INTERVAL: Duration = Duration::from_millis(500);
struct Harness {
pub workers: Vec<FakeWorker>,
pub registry: Arc<WorkerRegistry>,
pub observer: BatcherObserver,
pub delivery_log: DeliveryLog,
pub shutdown: CancellationToken,
task: Option<tokio::task::JoinHandle<()>>,
_probe_token: CancellationToken,
topic: String,
group_id: String,
pub ledger: Arc<TopicOffsetLedger>,
debug_recorder: Arc<DebugRecorder>,
deferred_flush_timeout: Duration,
pack_targets: PackTargets,
budget: Option<RequestBudget>,
partition_budget: BudgetLimits,
in_flight_limit: ChargeLimit,
revoke_drain_timeout: Option<Duration>,
rebalance_mode: RebalanceMode,
}
/// Production runs `group.protocol=consumer` (KIP-848), which also revokes
/// incrementally. The test broker, Redpanda v25.1.9, does not implement
/// ConsumerGroupHeartbeat (API key 68) and leaves the key out of its
/// ApiVersions response. So no case covers KIP-848; `Cooperative` takes the
/// same `incremental_unassign` path in the consumer.
#[derive(Clone, Copy, Debug, PartialEq)]
enum RebalanceMode {
Eager,
/// `cooperative-sticky`: a revoke takes only the partitions that move.
/// The broker is `Stable` while the old owner holds them, and the new owner
/// gets them in a second rebalance after the unassign.
Cooperative,
}
impl RebalanceMode {
fn configure(self, config: &mut ClientConfig) {
match self {
Self::Eager => {}
Self::Cooperative => {
config.set("partition.assignment.strategy", "cooperative-sticky");
}
}
}
}
// Short session and heartbeat so group handovers finish quickly.
fn make_kafka_consumer(
topic: &str,
group_id: &str,
instance_id: Option<&str>,
) -> StreamConsumer<RebalanceContext> {
make_kafka_consumer_with_context(
topic,
group_id,
instance_id,
RebalanceContext::new(),
RebalanceMode::Eager,
)
}
fn make_kafka_consumer_with_context(
topic: &str,
group_id: &str,
instance_id: Option<&str>,
context: RebalanceContext,
rebalance_mode: RebalanceMode,
) -> StreamConsumer<RebalanceContext> {
let mut config = ClientConfig::new();
config
.set("bootstrap.servers", KAFKA_BROKERS)
.set("group.id", group_id)
.set("auto.offset.reset", "earliest")
.set("enable.auto.commit", "false")
.set("enable.auto.offset.store", "false")
.set("session.timeout.ms", "6000")
.set("heartbeat.interval.ms", "1000")
.set("socket.timeout.ms", "5000");
if let Some(id) = instance_id {
config.set("group.instance.id", id);
}
rebalance_mode.configure(&mut config);
let kafka_consumer: StreamConsumer<RebalanceContext> =
config.create_with_context(context).expect("kafka consumer");
kafka_consumer.subscribe(&[topic]).expect("subscribe");
kafka_consumer
}
/// Each FakeWorker serves gRPC on its HTTP port + 1; a 10 s ack timeout (production: 60 s) so a stalled request fences quickly.
fn test_transport() -> GrpcTransport {
GrpcTransport::new(GrpcPort::OffsetFromHttp(1), Duration::from_secs(10))
}
/// Ticks every 100 ms, against 1 s in production.
fn spawn_reaper(
registry: Arc<WorkerRegistry>,
transport: Arc<GrpcTransport>,
observer: BatcherObserver,
token: CancellationToken,
) {
tokio::spawn(async move {
loop {
tokio::select! {
_ = token.cancelled() => break,
_ = tokio::time::sleep(Duration::from_millis(100)) => {}
}
for worker in registry.draining_workers() {
if !observer.has_in_flight(&worker) {
registry.complete_drain(&worker);
}
}
for worker in registry.reapable_workers() {
registry.remove_worker(&worker);
transport.remove_worker(&worker);
}
}
});
}
impl Harness {
async fn start(
topic: &str,
partitions: i32,
worker_count: usize,
deferred_flush_timeout: Duration,
registry_config: WorkerRegistryConfig,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
deferred_flush_timeout,
registry_config,
0,
ComponentOptions::new(),
common::PRODUCTION_PACK_TARGETS,
None,
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
/// One worker; every request carries a 500 ms soft budget.
async fn start_budgeted(topic: &str, partitions: i32) -> Self {
Self::start_budgeted_draining_revokes(topic, partitions, None).await
}
async fn start_budgeted_draining_revokes(
topic: &str,
partitions: i32,
revoke_drain_timeout: Option<Duration>,
) -> Self {
Self::start_inner(
topic,
partitions,
1,
Duration::from_secs(60),
fast_registry_config(),
0,
ComponentOptions::new(),
// A request waits for four events per partition, which one
// `alternating_visitors` call per partition fills.
PackTargets {
events: NonZeroUsize::new(4 * partitions as usize),
bytes: None,
latency_budget: Duration::from_secs(5),
},
Some(RequestBudget {
soft_budget: Duration::from_millis(500),
attempts: BUDGET_ATTEMPTS,
}),
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
revoke_drain_timeout,
RebalanceMode::Eager,
)
.await
}
async fn start_with_pack_targets(
topic: &str,
partitions: i32,
worker_count: usize,
deferred_flush_timeout: Duration,
registry_config: WorkerRegistryConfig,
pack_targets: PackTargets,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
deferred_flush_timeout,
registry_config,
0,
ComponentOptions::new(),
pack_targets,
None,
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
async fn start_with_liveness(
topic: &str,
worker_count: usize,
deferred_flush_timeout: Duration,
liveness_deadline: Duration,
stall_threshold: u32,
) -> Self {
Self::start_inner(
topic,
1,
worker_count,
deferred_flush_timeout,
fast_registry_config(),
0,
ComponentOptions::new()
.with_liveness_deadline(liveness_deadline)
.with_stall_threshold(stall_threshold),
common::ONE_MESSAGE_PER_REQUEST,
None,
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
async fn start_draining_revokes(
topic: &str,
partitions: i32,
worker_count: usize,
revoke_drain_timeout: Duration,
rebalance_mode: RebalanceMode,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
Duration::from_secs(60),
fast_registry_config(),
0,
ComponentOptions::new(),
common::PRODUCTION_PACK_TARGETS,
None,
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
Some(revoke_drain_timeout),
rebalance_mode,
)
.await
}
async fn start_byte_capped(topic: &str, batch_size_bytes: usize) -> Self {
Self::start_inner(
topic,
1,
1,
Duration::from_secs(60),
fast_registry_config(),
batch_size_bytes,
ComponentOptions::new(),
common::PRODUCTION_PACK_TARGETS,
None,
COMMIT_INTERVAL,
BudgetLimits::unset(),
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
async fn start_committing_every(
pack_targets: PackTargets,
topic: &str,
partitions: i32,
worker_count: usize,
commit_interval: Duration,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
Duration::from_secs(60),
fast_registry_config(),
0,
ComponentOptions::new(),
pack_targets,
None,
commit_interval,
BudgetLimits::unset(),
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
async fn start_with_partition_budget(
pack_targets: PackTargets,
topic: &str,
partitions: i32,
worker_count: usize,
partition_budget: BudgetLimits,
) -> Self {
Self::start_with_limits(
pack_targets,
topic,
partitions,
worker_count,
partition_budget,
ChargeLimit::new(NonZeroU64::new(1_000), NonZeroU64::new(UNBOUNDED_BYTES)),
)
.await
}
async fn start_with_limits(
pack_targets: PackTargets,
topic: &str,
partitions: i32,
worker_count: usize,
partition_budget: BudgetLimits,
in_flight_limit: ChargeLimit,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
Duration::from_secs(60),
fast_registry_config(),
0,
ComponentOptions::new(),
pack_targets,
None,
COMMIT_INTERVAL,
partition_budget,
in_flight_limit,
None,
RebalanceMode::Eager,
)
.await
}
async fn start_with_partition_budget_committing_every(
pack_targets: PackTargets,
topic: &str,
partitions: i32,
worker_count: usize,
partition_budget: BudgetLimits,
commit_interval: Duration,
) -> Self {
Self::start_inner(
topic,
partitions,
worker_count,
Duration::from_secs(60),
fast_registry_config(),
0,
ComponentOptions::new(),
pack_targets,
None,
commit_interval,
partition_budget,
ChargeLimit::unset(),
None,
RebalanceMode::Eager,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn start_inner(
topic: &str,
partitions: i32,
worker_count: usize,
deferred_flush_timeout: Duration,
registry_config: WorkerRegistryConfig,
batch_size_bytes: usize,
component_options: ComponentOptions,
pack_targets: PackTargets,
budget: Option<RequestBudget>,
commit_interval: Duration,
partition_budget: BudgetLimits,
in_flight_limit: ChargeLimit,
revoke_drain_timeout: Option<Duration>,
rebalance_mode: RebalanceMode,
) -> Self {
create_topic(topic, partitions).await;
let delivery_log: DeliveryLog = Arc::new(Mutex::new(Vec::new()));
let mut workers = Vec::new();
for _ in 0..worker_count {
workers.push(FakeWorker::start_logged(Arc::clone(&delivery_log)).await);
}
let worker_urls: Vec<String> = workers.iter().map(|w| w.url.clone()).collect();
let registry = Arc::new(WorkerRegistry::new(&worker_urls, registry_config));
let probe_token = CancellationToken::new();
Arc::clone(®istry).start_probing(probe_token.clone());
let pool_source = WorkerPoolSource::new(Arc::clone(®istry), RoutingStrategy::default());
let registry_for_test = Arc::clone(®istry);
let transport = Arc::new(test_transport());
let mut manager = Manager::builder("e2e-test")
.with_trap_signals(false)
.with_health_poll_interval(Duration::from_millis(100))
.build();
let handle = manager.register("consumer", component_options);
let shutdown = handle.shutdown_token();
let _monitor = manager.monitor_background();
let group_id = format!("e2e-{}", Uuid::new_v4());
let ledger = Arc::new(TopicOffsetLedger::new());
let debug_recorder = DebugRecorder::new(10_000, Duration::from_secs(3600));
let kafka_consumer = make_kafka_consumer_with_context(
topic,
&group_id,
None,
RebalanceContext::new(),
rebalance_mode,
);
let (batcher, outputs) = common::batcher(
pool_source,
Arc::clone(&transport),
deferred_flush_timeout,
Duration::from_millis(200),
pack_targets,
budget,
);
let consumer = IngestionConsumer::from_parts(
kafka_consumer,
batcher,
outputs,
Arc::clone(&transport),
worker_urls,
IngestionConsumerOptions {
batch_size: 50,
batch_size_bytes,
batch_timeout: Duration::from_millis(100),
group_id: "e2e-test".to_string(),
debug_recorder: Some(Arc::clone(&debug_recorder)),
commit_interval,
ledger: Arc::clone(&ledger),
partition_budget,
in_flight_limit,
revoke_drain_timeout,
},
handle,
);
let observer = consumer.batcher_observer();
spawn_reaper(
Arc::clone(®istry),
transport,
observer.clone(),
probe_token.clone(),
);
let task = tokio::spawn(async move { consumer.process().await });
// Give the consumer time to connect and enter the poll loop.
tokio::time::sleep(Duration::from_millis(300)).await;
Self {
workers,
registry: registry_for_test,
observer,
delivery_log,
shutdown,
task: Some(task),
_probe_token: probe_token,
topic: topic.to_string(),
group_id,
ledger,
debug_recorder,
deferred_flush_timeout,
pack_targets,
budget,
partition_budget,
in_flight_limit,
revoke_drain_timeout,
rebalance_mode,
}
}
/// Message count of each poll the consumer dispatched, before the packer merges any.
fn poll_sizes(&self) -> Vec<usize> {
self.debug_recorder
.backlog()
.into_iter()
.filter_map(|event| match event.kind {
DebugEventKind::BatchDispatched { messages, .. } => Some(messages),
_ => None,
})
.collect()
}
fn dispatched_assignment_epochs(&self) -> Vec<u64> {
self.debug_recorder
.backlog()
.into_iter()
.filter_map(|event| match event.kind {
DebugEventKind::BatchDispatched {
assignment_epoch, ..
} => Some(assignment_epoch),
_ => None,
})
.collect()
}
/// Abort the consumer task: no shutdown, no drain, no final commit.
fn crash_consumer(&mut self) {
if let Some(task) = self.task.take() {
task.abort();
}
}
/// Model a pod restart: a new consumer stack in the same group, against the same workers.
async fn restart_consumer(&mut self, registry_config: WorkerRegistryConfig) {
let worker_urls: Vec<String> = self.workers.iter().map(|w| w.url.clone()).collect();
let registry = Arc::new(WorkerRegistry::new(&worker_urls, registry_config));
Arc::clone(®istry).start_probing(self._probe_token.clone());
let pool_source = WorkerPoolSource::new(Arc::clone(®istry), RoutingStrategy::default());
let transport = Arc::new(test_transport());
let mut manager = Manager::builder("e2e-test-restarted")
.with_trap_signals(false)
.build();
let handle = manager.register("consumer", ComponentOptions::new());
self.shutdown = handle.shutdown_token();
self.ledger = Arc::new(TopicOffsetLedger::new());
let kafka_consumer = make_kafka_consumer_with_context(
&self.topic,
&self.group_id,
None,
RebalanceContext::new(),
self.rebalance_mode,
);
let (batcher, outputs) = common::batcher(
pool_source,
Arc::clone(&transport),
self.deferred_flush_timeout,
Duration::from_millis(200),
self.pack_targets,
self.budget,
);
let consumer = IngestionConsumer::from_parts(
kafka_consumer,
batcher,
outputs,
Arc::clone(&transport),
worker_urls,
IngestionConsumerOptions {
batch_size: 50,
batch_size_bytes: 0,
batch_timeout: Duration::from_millis(100),
group_id: "e2e-test".to_string(),
debug_recorder: None,
commit_interval: COMMIT_INTERVAL,
ledger: Arc::clone(&self.ledger),
partition_budget: self.partition_budget,
in_flight_limit: self.in_flight_limit,
revoke_drain_timeout: self.revoke_drain_timeout,
},
handle,
);
self.observer = consumer.batcher_observer();
spawn_reaper(
Arc::clone(®istry),
transport,
self.observer.clone(),
self._probe_token.clone(),
);
self.registry = registry;
self.task = Some(tokio::spawn(async move { consumer.process().await }));
// Give the restarted consumer time to rejoin the group and start polling.
tokio::time::sleep(Duration::from_millis(300)).await;
}
async fn wait_for_consumer_exit(&mut self, timeout: Duration) -> bool {
let Some(task) = self.task.take() else {
return true;
};
tokio::time::timeout(timeout, task).await.is_ok()
}
fn committed_offset(&self, partition: i32) -> Option<i64> {
broker_committed_offset(&self.group_id, &self.topic, partition)
}
async fn wait_for(&self, total: usize, timeout: Duration) {
let deadline = tokio::time::Instant::now() + timeout;
loop {
let got: usize = self.workers.iter().map(|w| w.count()).sum();
if got >= total {
return;
}
assert!(
tokio::time::Instant::now() < deadline,
"timed out waiting for {total} messages (got {got})"
);
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
async fn stop(mut self) {
self.shutdown.cancel();
if let Some(task) = self.task.take() {
let _ = tokio::time::timeout(Duration::from_secs(3), task).await;
}
}
/// A second consumer in the same group, over the same workers, as a second pod.
async fn join_second_consumer(&self) -> SecondConsumer {
let worker_urls: Vec<String> = self.workers.iter().map(|w| w.url.clone()).collect();
let registry = Arc::new(WorkerRegistry::new(&worker_urls, fast_registry_config()));
let probe = CancellationToken::new();
Arc::clone(®istry).start_probing(probe.clone());
let pool_source = WorkerPoolSource::new(Arc::clone(®istry), RoutingStrategy::default());
let transport = Arc::new(test_transport());
let mut manager = Manager::builder("e2e-c2").with_trap_signals(false).build();
let handle = manager.register("consumer", ComponentOptions::new());
let shutdown = handle.shutdown_token();
let (batcher, outputs) = common::batcher(
pool_source,
Arc::clone(&transport),
self.deferred_flush_timeout,
Duration::from_millis(200),
self.pack_targets,
self.budget,
);
let consumer = IngestionConsumer::from_parts(
make_kafka_consumer_with_context(
&self.topic,
&self.group_id,
None,
RebalanceContext::new(),
self.rebalance_mode,
),
batcher,
outputs,
transport,
worker_urls,
IngestionConsumerOptions {
batch_size: 50,
batch_size_bytes: 0,
batch_timeout: Duration::from_millis(100),
group_id: "e2e-test".to_string(),
debug_recorder: None,
commit_interval: COMMIT_INTERVAL,
ledger: Arc::new(TopicOffsetLedger::new()),
partition_budget: self.partition_budget,
in_flight_limit: self.in_flight_limit,
revoke_drain_timeout: self.revoke_drain_timeout,
},
handle,
);
let task = tokio::spawn(async move { consumer.process().await });
SecondConsumer {
shutdown,
probe,
task,
_manager: manager,
}
}
}
struct SecondConsumer {
shutdown: CancellationToken,
probe: CancellationToken,
task: tokio::task::JoinHandle<()>,
_manager: Manager,
}
impl SecondConsumer {
async fn stop(self) {
self.shutdown.cancel();
let _ = tokio::time::timeout(Duration::from_secs(5), self.task).await;
self.probe.cancel();
}
}
fn fast_registry_config() -> WorkerRegistryConfig {
WorkerRegistryConfig {
probe_interval: Duration::from_millis(50),
dead_declaration: Duration::from_millis(200),
passive_window: Duration::from_secs(30),
passive_error_threshold: 0.5,
passive_min_samples: 1,
degraded_hold: Duration::from_millis(100),
min_state_duration: Duration::ZERO,
probe_failure_threshold: 2,
drain_timeout: Duration::from_secs(5),
}
}
fn broker_committed_offset(group_id: &str, topic: &str, partition: i32) -> Option<i64> {
let probe: BaseConsumer = ClientConfig::new()
.set("bootstrap.servers", KAFKA_BROKERS)
.set("group.id", group_id)
.create()
.expect("committed-offset probe");
let mut tpl = TopicPartitionList::new();
tpl.add_partition(topic, partition);
let committed = probe
.committed_offsets(tpl, Duration::from_secs(2))
.expect("fetch committed offsets");
match committed.elements()[0].offset() {
rdkafka::Offset::Offset(offset) => Some(offset),
_ => None,
}
}
#[derive(Debug)]
struct GroupState {
state: String,
owned: Vec<usize>,
}
impl GroupState {
fn handed_over(&self) -> bool {
self.state == "Stable" && self.owned == [1, 1]
}
fn holds_a_revoke(&self, mode: RebalanceMode) -> bool {
match mode {
RebalanceMode::Eager => self.state == "PreparingRebalance" && self.owned.len() == 2,
RebalanceMode::Cooperative => self.state == "Stable" && self.owned == [0, 1],
}
}
}
fn group_state(group_id: &str) -> Option<GroupState> {
let probe: BaseConsumer = ClientConfig::new()
.set("bootstrap.servers", KAFKA_BROKERS)
.set("group.id", group_id)
.create()
.expect("group probe");
let groups = probe
.client()
.fetch_group_list(Some(group_id), Duration::from_secs(5))
.ok()?;
let group = groups
.groups()
.iter()
.find(|group| group.name() == group_id)?;
let mut owned: Vec<usize> = group
.members()
.iter()
.map(|member| assigned_partition_count(member.assignment().unwrap_or_default()))
.collect();
owned.sort_unstable();
Some(GroupState {
state: group.state().to_string(),
owned,
})
}
/// Counts the partitions in a classic-protocol `ConsumerProtocolAssignment`:
/// an i16 version, then an i32 count of (i16-length topic, i32 count of i32 partitions).
fn assigned_partition_count(assignment: &[u8]) -> usize {
fn read<const N: usize>(bytes: &[u8], at: &mut usize) -> Option<[u8; N]> {
let read = bytes.get(*at..*at + N)?.try_into().ok()?;
*at += N;
Some(read)
}
fn count(assignment: &[u8]) -> Option<usize> {
let mut at = 2;
let topics = i32::from_be_bytes(read(assignment, &mut at)?);
let mut partitions = 0;
for _ in 0..topics {
at += i16::from_be_bytes(read(assignment, &mut at)?) as usize;
let in_topic = i32::from_be_bytes(read(assignment, &mut at)?) as usize;
at += 4 * in_topic;
partitions += in_topic;
}
Some(partitions)
}
if assignment.is_empty() {
return 0;
}
count(assignment).expect("a well-formed consumer protocol assignment")
}
async fn wait_for_group(
group_id: &str,
timeout: Duration,
what: &str,
cond: impl Fn(&GroupState) -> bool,
) {
let deadline = tokio::time::Instant::now() + timeout;
loop {
let observed = group_state(group_id);
if observed.as_ref().is_some_and(&cond) {
return;
}
assert!(
tokio::time::Instant::now() < deadline,
"timed out waiting for the group to {what}, last seen {observed:?}"
);
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
async fn wait_until(timeout: Duration, msg: &str, mut cond: impl FnMut() -> bool) {
let deadline = tokio::time::Instant::now() + timeout;
while !cond() {
assert!(
tokio::time::Instant::now() < deadline,
"timed out waiting for: {msg}"
);
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
fn sole_arrived_worker(harness: &Harness) -> usize {
let arrived: Vec<usize> = harness
.workers
.iter()
.enumerate()
.filter(|(_, w)| w.arrived_count() > 0)
.map(|(i, _)| i)
.collect();
assert_eq!(
arrived.len(),
1,
"expected exactly one worker to have a batch in flight, got {arrived:?}"
);
arrived[0]
}
#[rstest]
#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]
#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]
#[tokio::test]
async fn messages_per_distinct_id_arrive_in_order(#[case] pack_targets: PackTargets) {
let topic = format!("e2e-ordering-{}", Uuid::new_v4());
let harness = Harness::start_with_pack_targets(
&topic,
3,
2,
Duration::from_secs(60),
fast_registry_config(),
pack_targets,
)
.await;
let producer = make_producer();
for seq in 0..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
produce(&producer, &topic, 2, "tok", "user-3", seq).await;
}
harness.wait_for(24, Duration::from_secs(15)).await;
let total: usize = harness.workers.iter().map(|w| w.count()).sum();
assert_eq!(total, 24, "expected 24 messages total, got {total}");
// A key can move between workers across requests, so order holds per worker.
for worker in &harness.workers {
for user in ["user-1", "user-2", "user-3"] {
let seqs = worker.seqs_for(user);
assert!(
seqs.windows(2).all(|w| w[0] < w[1]),
"{user} arrived out of order on a worker: {seqs:?}"
);
}
}
harness.stop().await;
}
#[tokio::test]
async fn a_partial_ack_redelivers_only_the_unfinished_messages() {
let topic = format!("e2e-partial-{}", Uuid::new_v4());
let harness = Harness::start_budgeted(&topic, 1).await;
*harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-a".to_string(), 1));
produce_keyed_in_one_transaction(
&topic,
"tok",
&alternating_visitors(0, "tok:203.0.113.7", 0),
)
.await;
// A commit means every resend settled, so a late duplicate is in the log.
wait_until(Duration::from_secs(15), "the offsets commit", || {
harness
.committed_offset(0)
.is_some_and(|offset| offset >= 4)
})
.await;
let worker = &harness.workers[0];
assert_eq!(
worker.batch_sizes()[0],
4,
"all four messages ride one request: sizes {:?}",
worker.batch_sizes()
);
assert_eq!(worker.seqs_for("visitor-a"), vec![0, 2]);
assert_eq!(worker.seqs_for("visitor-b"), vec![1, 3]);
assert_eq!(
*worker.replayed.lock().unwrap(),
vec![("visitor-a".to_string(), 0), ("visitor-a".to_string(), 2)],
"only the unfinished messages are redelivered, marked as replay"
);
harness.stop().await;
}
#[tokio::test]
async fn a_crash_before_a_partial_acks_resend_loses_nothing() {
let topic = format!("e2e-partial-crash-{}", Uuid::new_v4());
let mut harness = Harness::start_budgeted(&topic, 1).await;
*harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-a".to_string(), 1));
// The resend never settles, so it is still in flight at the crash.
harness.workers[0]
.hold_replays
.store(true, Ordering::SeqCst);
produce_keyed_in_one_transaction(
&topic,
"tok",
&alternating_visitors(0, "tok:203.0.113.7", 0),
)
.await;
wait_until(
Duration::from_secs(15),
"the resend to reach the worker",
|| harness.workers[0].arrived_count() >= 2,
)
.await;
assert!(
harness.workers[0].seqs_for("visitor-a").is_empty(),
"visitor A's messages are unfinished"
);
// Commits are async: wait so a wrong commit lands before the crash.
tokio::time::sleep(Duration::from_millis(500)).await;
assert_eq!(
harness.committed_offset(0),
None,
"nothing commits while an unfinished message waits for its resend"
);
harness.crash_consumer();
harness.restart_consumer(fast_registry_config()).await;
wait_until(
Duration::from_secs(30),
"the offsets to commit after the restart",
|| {
harness
.committed_offset(0)
.is_some_and(|offset| offset >= 4)
},
)
.await;
assert_eq!(
harness.workers[0].seqs_for("visitor-a"),
vec![0, 2],
"the restarted consumer delivers the unfinished messages"
);
harness.stop().await;
}
#[tokio::test]
async fn a_partial_ack_after_a_revoke_resends_nothing() {
let topic = format!("e2e-partial-revoke-{}", Uuid::new_v4());
let harness = Harness::start_budgeted(&topic, 2).await;
*harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-a".to_string(), 1));
let partitions = [
TopicPartition::new(topic.clone(), 0),
TopicPartition::new(topic.clone(), 1),
];
let records: Vec<_> = alternating_visitors(0, "tok:203.0.113.7", 0)
.into_iter()
.chain(alternating_visitors(1, "tok:203.0.113.8", 10))
.collect();
// A trickle keeps the consumer polling, so it takes part in the rebalance.
let producer = make_producer();
producer
.client()
.fetch_metadata(Some(&topic), Duration::from_secs(5))
.expect("trickle producer metadata");
// Hold the first request; the trickle's requests queue behind it (one un-acked per worker).
let gate = harness.workers[0].block().await;
produce_keyed_in_one_transaction(&topic, "tok", &records).await;
let trickling = Arc::new(AtomicBool::new(true));
let trickle = tokio::spawn({
let trickling = Arc::clone(&trickling);
let topic = topic.clone();
async move {
let mut seq = 1000usize;
while trickling.load(Ordering::SeqCst) {
produce(&producer, &topic, 0, "tok", "trickle", seq).await;
seq += 1;
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
});
wait_until(
Duration::from_secs(15),
"a request to reach the worker",
|| harness.workers[0].arrived_count() > 0,
)
.await;
let generations: Vec<u64> = partitions
.iter()
.map(|partition| harness.ledger.generation(partition))
.collect();
let second = harness.join_second_consumer().await;
// Wait under the 10 s ack timeout, after which the held request returns as a replay.
wait_until(
Duration::from_secs(8),
"the rebalance to revoke both partitions",
|| {
partitions
.iter()
.zip(&generations)
.all(|(partition, before)| harness.ledger.generation(partition) > *before)
},
)
.await;
trickling.store(false, Ordering::SeqCst);
trickle.await.expect("trickle task");
drop(gate);
wait_until(
Duration::from_secs(30),
"every message to be delivered",
|| {
let delivered: HashSet<(String, usize)> = harness
.delivery_log
.lock()
.unwrap()
.iter()
.cloned()
.collect();
records
.iter()
.all(|(_, _, visitor, seq)| delivered.contains(&(visitor.to_string(), *seq)))
},
)
.await;
// The old owner may not have settled the partial ack yet; wait for it.
wait_until(
Duration::from_secs(15),
"the old owner to settle the partial ack",
|| harness.observer.total_in_flight() == 0,
)
.await;
assert!(
harness.workers[0]
.leave_unfinished
.lock()
.unwrap()
.is_none(),
"the held request was acked as partial"
);
assert_eq!(
*harness.workers[0].replayed.lock().unwrap(),
Vec::<(String, usize)>::new(),
"the old owner resends nothing for a revoked partition"
);
second.stop().await;
harness.stop().await;
}
#[tokio::test]
async fn a_partial_ack_during_a_drain_commits_only_what_was_processed() {
let topic = format!("e2e-partial-drain-{}", Uuid::new_v4());
let harness =
Harness::start_budgeted_draining_revokes(&topic, 1, Some(Duration::from_secs(20))).await;
// The partition holds a, b, a, b, and the b messages come back unfinished.
*harness.workers[0].leave_unfinished.lock().unwrap() = Some(("visitor-b".to_string(), 1));
let records = alternating_visitors(0, "tok:203.0.113.7", 0);
// A trickle keeps the consumer polling, so it takes part in the rebalance.
let producer = make_producer();
producer
.client()
.fetch_metadata(Some(&topic), Duration::from_secs(5))
.expect("trickle producer metadata");
let gate = harness.workers[0].block().await;
produce_keyed_in_one_transaction(&topic, "tok", &records).await;
let trickling = Arc::new(AtomicBool::new(true));
let trickle = tokio::spawn({
let trickling = Arc::clone(&trickling);
let topic = topic.clone();
async move {
let mut seq = 1000usize;
while trickling.load(Ordering::SeqCst) {
produce(&producer, &topic, 0, "tok", "trickle", seq).await;
seq += 1;
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
});
wait_until(
Duration::from_secs(15),
"a request to reach the worker",
|| harness.workers[0].arrived_count() > 0,
)
.await;
let second = harness.join_second_consumer().await;
// The drain waits on the gated request.
tokio::time::sleep(Duration::from_secs(2)).await;
trickling.store(false, Ordering::SeqCst);
trickle.await.expect("trickle task");
drop(gate);
wait_until(
Duration::from_secs(30),
"every message to be delivered",
|| {
let delivered: HashSet<(String, usize)> = harness
.delivery_log
.lock()
.unwrap()
.iter()
.cloned()
.collect();
records
.iter()
.all(|(_, _, visitor, seq)| delivered.contains(&(visitor.to_string(), *seq)))
},
)
.await;
// Time for a replay by the new owner to arrive.
tokio::time::sleep(Duration::from_secs(2)).await;
assert!(
harness.workers[0]
.leave_unfinished
.lock()
.unwrap()
.is_none(),
"the gated request was acked as partial"
);
assert_eq!(
*harness.workers[0].replayed.lock().unwrap(),
Vec::<(String, usize)>::new(),
"the old owner resends nothing of a draining partition"
);
let log = harness.delivery_log.lock().unwrap().clone();
let mut seen = HashSet::new();
let processed_twice: BTreeSet<(String, usize)> = log
.iter()
.filter(|entry| entry.0 != "trickle" && !seen.insert(*entry))
.cloned()
.collect();
assert_eq!(
processed_twice,
BTreeSet::from([("visitor-a".to_string(), 2)])
);
second.stop().await;
harness.stop().await;
}
#[tokio::test]
async fn a_message_that_never_fits_the_budget_lands_without_one() {
let topic = format!("e2e-partial-escalate-{}", Uuid::new_v4());
let harness = Harness::start_budgeted(&topic, 1).await;
*harness.workers[0].leave_unfinished.lock().unwrap() =
Some(("visitor-a".to_string(), usize::MAX));
produce_keyed_in_one_transaction(
&topic,
"tok",
&alternating_visitors(0, "tok:203.0.113.7", 0),
)
.await;
wait_until(Duration::from_secs(15), "the offsets commit", || {
harness
.committed_offset(0)
.is_some_and(|offset| offset >= 4)
})
.await;
let worker = &harness.workers[0];
assert_eq!(worker.seqs_for("visitor-a"), vec![0, 2]);
assert_eq!(worker.seqs_for("visitor-b"), vec![1, 3]);
assert_eq!(
worker.batch_sizes(),
vec![4, 2, 2],
"visitor A goes out once more with a budget, then once without"
);
harness.stop().await;
}
#[tokio::test]
async fn byte_bound_caps_batches_below_the_count_bound() {
let topic = format!("e2e-bytecap-{}", Uuid::new_v4());
let harness = Harness::start_byte_capped(&topic, 4096).await;
let producer = make_producer();
let payload = vec![b'x'; 2048];
for seq in 0..12usize {
produce_raw(
&producer,
&topic,
0,
&format!("tok:user-1:{seq}"),
&payload,
None,
)
.await;
}
harness.wait_for(12, Duration::from_secs(20)).await;
let sizes = harness.poll_sizes();
assert_eq!(
sizes.iter().sum::<usize>(),
12,
"every message must still be polled, got polls {sizes:?}"
);
// A batch timeout only shrinks polls, so producer timing cannot break this.
assert!(
sizes.iter().all(|&n| n <= 2),
"no poll may exceed the byte bound by more than one message, got {sizes:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn message_larger_than_byte_bound_is_still_delivered() {
let topic = format!("e2e-bytecap-oversize-{}", Uuid::new_v4());
let harness = Harness::start_byte_capped(&topic, 4096).await;
let producer = make_producer();
produce_raw(
&producer,
&topic,
0,
"tok:user-1:0",
&vec![b'x'; 16384],
None,
)
.await;
harness.wait_for(1, Duration::from_secs(20)).await;
assert_eq!(
harness.workers[0].batch_sizes(),
vec![1],
"the oversized message must arrive as a batch of one"
);
harness.stop().await;
}
#[tokio::test]
async fn failing_worker_triggers_rerouting() {
let topic = format!("e2e-failover-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
harness.wait_for(8, Duration::from_secs(10)).await;
let dead_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {
0
} else {
1
};
let live_idx = 1 - dead_idx;
harness.workers[dead_idx]
.healthy
.store(false, Ordering::Relaxed);
tokio::time::sleep(Duration::from_millis(600)).await;
for seq in 4..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
harness.wait_for(12, Duration::from_secs(10)).await;
let live_seqs = harness.workers[live_idx].seqs_for("user-1");
let batch2_seqs: Vec<usize> = live_seqs.iter().copied().filter(|&s| s >= 4).collect();
assert_eq!(
batch2_seqs,
vec![4, 5, 6, 7],
"batch-2 messages for user-1 missing or out-of-order on live worker: {live_seqs:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn failed_sends_degrade_passive_health() {
let topic = format!("e2e-passive-fault-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
1,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
harness.workers[0].ingest_ok.store(false, Ordering::SeqCst);
produce(&producer, &topic, 0, "tok", "user-1", 0).await;
let worker = harness.workers[0].url.clone();
wait_until(
Duration::from_secs(10),
"failed sends to degrade the worker",
|| harness.registry.state(&worker) != WorkerState::Healthy,
)
.await;
harness.workers[0].ingest_ok.store(true, Ordering::SeqCst);
harness.stop().await;
}
#[tokio::test]
async fn draining_worker_takes_no_new_work_and_reroutes_in_order() {
let topic = format!("e2e-drain-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
harness.wait_for(8, Duration::from_secs(10)).await;
let drain_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {
0
} else {
1
};
let live_idx = 1 - drain_idx;
let drain_url = harness.workers[drain_idx].url.clone();
harness.registry.start_draining(&drain_url);
wait_until(Duration::from_secs(2), "idle drainer to be reaped", || {
!harness
.registry
.workers()
.iter()
.any(|w| w.as_ref() == drain_url)
})
.await;
for seq in 4..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
harness.wait_for(12, Duration::from_secs(10)).await;
let mut all_user1: Vec<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for("user-1"))
.collect();
all_user1.sort_unstable();
assert_eq!(
all_user1,
(0..8).collect::<Vec<_>>(),
"user-1 messages were lost or duplicated: {all_user1:?}"
);
let drained_seqs = harness.workers[drain_idx].seqs_for("user-1");
assert!(
drained_seqs.iter().all(|&s| s < 4),
"draining worker received new work: {drained_seqs:?}"
);
let live_batch2: Vec<usize> = harness.workers[live_idx]
.seqs_for("user-1")
.into_iter()
.filter(|&s| s >= 4)
.collect();
assert_eq!(
live_batch2,
vec![4, 5, 6, 7],
"batch-2 messages misrouted or out of order on the live worker: {live_batch2:?}"
);
harness.stop().await;
}
/// Whether `url` is currently routable (healthy or degraded) per the registry.
fn in_pool(harness: &Harness, url: &str) -> bool {
harness
.registry
.healthy_workers()
.iter()
.any(|w| w.as_ref() == url)
}
#[tokio::test]
async fn send_failure_mid_flight_replays_to_survivor_in_order() {
let topic = format!("e2e-replay-single-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Freeze both workers so whichever gets the batch holds it in flight.
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let failed = sole_arrived_worker(&harness);
let survivor = 1 - failed;
harness.workers[failed]
.healthy
.store(false, Ordering::SeqCst);
guards[survivor] = None;
guards[failed] = None;
harness.wait_for(4, Duration::from_secs(10)).await;
assert_eq!(
harness.workers[failed].count(),
0,
"the failed worker recorded nothing (it only returned errors)"
);
assert_eq!(
harness.workers[survivor].seqs_for("user-1"),
vec![0, 1, 2, 3],
"user-1 must replay to the survivor in Kafka order"
);
harness.stop().await;
}
#[tokio::test]
async fn partial_send_failure_replays_only_the_failed_subbatch() {
let topic = format!("e2e-replay-partial-{}", Uuid::new_v4());
let one_key_of_four_per_request = PackTargets {
events: NonZeroUsize::new(4),
bytes: None,
latency_budget: Duration::ZERO,
};
let harness = Harness::start_with_pack_targets(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
one_key_of_four_per_request,
)
.await;
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
let records: Vec<(&str, usize)> = (0..4usize)
.flat_map(|seq| [("user-1", seq), ("user-2", seq)])
.collect();
produce_in_one_transaction(&topic, 0, "tok", &records).await;
wait_until(
Duration::from_secs(10),
"both workers to receive a sub-batch",
|| harness.workers.iter().all(|w| w.arrived_count() > 0),
)
.await;
harness.workers[0].healthy.store(false, Ordering::SeqCst);
guards[1] = None;
guards[0] = None;
harness.wait_for(8, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[0].count(),
0,
"failed worker recorded nothing"
);
assert_eq!(
harness.workers[1].seqs_for("user-1"),
vec![0, 1, 2, 3],
"user-1 fully on the survivor in order"
);
assert_eq!(
harness.workers[1].seqs_for("user-2"),
vec![0, 1, 2, 3],
"user-2 (the originally-successful sub-batch) intact, not duplicated"
);
harness.stop().await;
}
#[tokio::test]
async fn deferred_flush_retries_until_a_worker_recovers() {
let topic = format!("e2e-replay-wait-{}", Uuid::new_v4());
let liveness_deadline = Duration::from_millis(1500);
let harness =
Harness::start_with_liveness(&topic, 2, Duration::from_secs(60), liveness_deadline, 2)
.await;
let producer = make_producer();
// Take worker 1 out of the pool so the batch routes to worker 0.
harness.workers[1].healthy.store(false, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to leave the pool",
|| !in_pool(&harness, &harness.workers[1].url),
)
.await;
let guard0 = harness.workers[0].block().await;
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach worker 0", || {
harness.workers[0].arrived_count() > 0
})
.await;
// Fail worker 0 too: now there is nowhere to replay.
harness.workers[0].healthy.store(false, Ordering::SeqCst);
drop(guard0);
// Only the oldest poll is stashed, so read its count rather than assert 4.
wait_until(
Duration::from_secs(10),
"worker 0 to leave the pool",
|| !in_pool(&harness, &harness.workers[0].url),
)
.await;
wait_until(
Duration::from_secs(10),
"the failed send to be deferred",
|| harness.observer.pending_messages() > 0,
)
.await;
let held = harness.observer.pending_messages();
tokio::time::sleep(liveness_deadline * 4).await;
assert_eq!(
harness.observer.pending_messages(),
held,
"deferred work must be held steady while no worker is available"
);
assert!(
!harness.shutdown.is_cancelled(),
"waiting for a worker must keep the liveness heartbeat going"
);
// Worker 1 rejoins blocked, so the flush waits on it past the liveness deadline.
let guard1 = harness.workers[1].block().await;
harness.workers[1].healthy.store(true, Ordering::SeqCst);
wait_until(Duration::from_secs(10), "replay to reach worker 1", || {
harness.workers[1].arrived_count() > 0
})
.await;
tokio::time::sleep(liveness_deadline * 4).await;
assert!(
!harness.shutdown.is_cancelled(),
"waiting on a slow worker must keep the liveness heartbeat going"
);
drop(guard1);
harness.wait_for(4, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[1].seqs_for("user-1"),
vec![0, 1, 2, 3],
"deferred user-1 must flush to the recovered worker in order"
);
assert_eq!(
harness.workers[0].count(),
0,
"failed worker recorded nothing"
);
harness.stop().await;
}
#[tokio::test]
async fn no_worker_exit_comes_from_the_flush_timeout_not_a_liveness_stall() {
let topic = format!("e2e-flush-timeout-liveness-{}", Uuid::new_v4());
let flush_timeout = Duration::from_secs(4);
let liveness_deadline = Duration::from_millis(1500);
let mut harness =
Harness::start_with_liveness(&topic, 2, flush_timeout, liveness_deadline, 2).await;
let producer = make_producer();
harness.workers[1].healthy.store(false, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to leave the pool",
|| !in_pool(&harness, &harness.workers[1].url),
)
.await;
let guard0 = harness.workers[0].block().await;
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach worker 0", || {
harness.workers[0].arrived_count() > 0
})
.await;
harness.workers[0].healthy.store(false, Ordering::SeqCst);
drop(guard0);
wait_until(
Duration::from_secs(10),
"the failed send to be deferred",
|| harness.observer.pending_messages() > 0,
)
.await;
let deferred_at = tokio::time::Instant::now();
assert!(
harness
.wait_for_consumer_exit(Duration::from_secs(20))
.await,
"consumer must fail the batch once nothing lands for a full flush timeout"
);
let waited = deferred_at.elapsed();
assert!(
waited >= flush_timeout * 3 / 4,
"consumer exited after {waited:?}: a liveness stall, not the {flush_timeout:?} flush timeout"
);
assert_eq!(
harness.workers.iter().map(|w| w.count()).sum::<usize>(),
0,
"nothing should be delivered"
);
}
#[tokio::test]
async fn slow_deferred_drain_with_progress_outlasts_the_flush_timeout() {
let topic = format!("e2e-slow-drain-{}", Uuid::new_v4());
let flush_timeout = Duration::from_secs(3);
let liveness_deadline = Duration::from_millis(1500);
let harness =
Harness::start_with_liveness(&topic, 3, flush_timeout, liveness_deadline, 2).await;
let producer = make_producer();
// Workers 1 and 2 out, so both keys route to worker 0 together.
for i in [1, 2] {
harness.workers[i].healthy.store(false, Ordering::SeqCst);
}
wait_until(
Duration::from_secs(10),
"workers 1 and 2 to leave the pool",
|| {
!in_pool(&harness, &harness.workers[1].url)
&& !in_pool(&harness, &harness.workers[2].url)
},
)
.await;
let guard0 = harness.workers[0].block().await;
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 0, "tok", "user-2", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach worker 0", || {
harness.workers[0].arrived_count() > 0
})
.await;
// Rejoin workers 1 and 2 before worker 0 fails, so the flush spreads the keys over them.
let mut guard1 = Some(harness.workers[1].block().await);
let guard2 = harness.workers[2].block().await;
for i in [1, 2] {
harness.workers[i].healthy.store(true, Ordering::SeqCst);
}
wait_until(
Duration::from_secs(10),
"workers 1 and 2 to rejoin the pool",
|| in_pool(&harness, &harness.workers[1].url) && in_pool(&harness, &harness.workers[2].url),
)
.await;
harness.workers[0].healthy.store(false, Ordering::SeqCst);
drop(guard0);
wait_until(
Duration::from_secs(10),
"the flush to reach both workers",
|| harness.workers[1].arrived_count() > 0 && harness.workers[2].arrived_count() > 0,
)
.await;
let flush_started = tokio::time::Instant::now();
// Worker 2 fails its key now, so that key re-defers into a later round.
harness.workers[2].ingest_ok.store(false, Ordering::SeqCst);
drop(guard2);
wait_until(Duration::from_secs(10), "worker 2 to answer", || {
harness.workers[2].arrived_count() == 1 && harness.observer.pending_messages() > 0
})
.await;
let guard2 = harness.workers[2].block().await;
harness.workers[2].ingest_ok.store(true, Ordering::SeqCst);
// Worker 1 lands its key only after the original flush deadline has passed.
tokio::time::sleep(flush_timeout + Duration::from_millis(200)).await;
assert!(
!harness.shutdown.is_cancelled(),
"a slow in-flight send must not trip the liveness stall"
);
guard1.take();
wait_until(
Duration::from_secs(10),
"the retried key to reach worker 2",
|| harness.workers[2].arrived_count() == 2,
)
.await;
// The second round waits past the liveness deadline again before landing.
tokio::time::sleep(liveness_deadline * 3).await;
assert!(
!harness.shutdown.is_cancelled(),
"the second flush round must keep the heartbeat going"
);
drop(guard2);
harness.wait_for(8, Duration::from_secs(15)).await;
assert!(
flush_started.elapsed() > flush_timeout,
"the drain must have outlasted the flush timeout for this test to mean anything"
);
assert!(
!harness.shutdown.is_cancelled(),
"the consumer must still be running"
);
let log = harness.delivery_log.lock().unwrap().clone();
for user in ["user-1", "user-2"] {
let seqs: Vec<usize> = log
.iter()
.filter(|(did, _)| did == user)
.map(|(_, seq)| *seq)
.collect();
assert_eq!(
seqs,
vec![0, 1, 2, 3],
"{user} must land exactly once, in order; log: {log:?}"
);
}
assert_eq!(
harness.workers[0].count(),
0,
"failed worker recorded nothing"
);
harness.stop().await;
}
#[rstest]
#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]
#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]
#[tokio::test]
async fn multi_key_send_failure_replays_every_key_in_order(#[case] pack_targets: PackTargets) {
let topic = format!("e2e-replay-multikey-{}", Uuid::new_v4());
let harness = Harness::start_with_pack_targets(
&topic,
3,
2,
Duration::from_secs(60),
fast_registry_config(),
pack_targets,
)
.await;
let producer = make_producer();
// Worker 1 out so all three keys land on worker 0.
harness.workers[1].healthy.store(false, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to leave the pool",
|| !in_pool(&harness, &harness.workers[1].url),
)
.await;
let guard0 = harness.workers[0].block().await;
for seq in 0..3usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
produce(&producer, &topic, 2, "tok", "user-3", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach worker 0", || {
harness.workers[0].arrived_count() > 0
})
.await;
harness.workers[0].healthy.store(false, Ordering::SeqCst);
harness.workers[1].healthy.store(true, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to rejoin the pool",
|| in_pool(&harness, &harness.workers[1].url),
)
.await;
drop(guard0);
harness.wait_for(9, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[0].count(),
0,
"failed worker recorded nothing"
);
for user in ["user-1", "user-2", "user-3"] {
assert_eq!(
harness.workers[1].seqs_for(user),
vec![0, 1, 2],
"{user} must replay to the recovered worker in order"
);
}
harness.stop().await;
}
#[tokio::test]
async fn flush_target_failure_re_defers_then_replays_in_order() {
let topic = format!("e2e-replay-cascade-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let first = sole_arrived_worker(&harness);
let second = 1 - first;
// Fail the first send; the replay goes to the other, blocked worker.
harness.workers[first]
.healthy
.store(false, Ordering::SeqCst);
guards[first] = None;
wait_until(
Duration::from_secs(10),
"the replay to reach the second worker",
|| harness.workers[second].arrived_count() > 0,
)
.await;
// Fail the replay too → it must be re-deferred, not lost.
harness.workers[second]
.healthy
.store(false, Ordering::SeqCst);
guards[second] = None;
wait_until(
Duration::from_secs(10),
"both workers to leave the pool",
|| {
!in_pool(&harness, &harness.workers[first].url)
&& !in_pool(&harness, &harness.workers[second].url)
},
)
.await;
// The failed replay can wait in reconnect backoff before it reaches the stash.
wait_until(
Duration::from_secs(10),
"re-deferred work to be held after the replay target also fails",
|| harness.observer.pending_messages() > 0,
)
.await;
harness.workers[second]
.healthy
.store(true, Ordering::SeqCst);
harness.wait_for(4, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[first].count(),
0,
"the worker that only ever failed recorded nothing"
);
assert_eq!(
harness.workers[second].seqs_for("user-1"),
vec![0, 1, 2, 3],
"user-1 replays to the recovered worker in order despite the cascade"
);
harness.stop().await;
}
#[tokio::test]
async fn full_pool_drain_holds_fresh_keys_until_a_worker_returns() {
let topic = format!("e2e-full-drain-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// The whole pool drains before any message arrives.
for w in &harness.workers {
harness.registry.start_draining(&w.url);
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
// Let the consumer collect and process the batch while nothing is routable.
tokio::time::sleep(Duration::from_millis(800)).await;
assert_eq!(
harness.workers.iter().map(|w| w.count()).sum::<usize>(),
0,
"nothing can be delivered while the whole pool is drained"
);
harness
.registry
.add_worker(WorkerId::from(harness.workers[0].url.as_str()));
harness.wait_for(4, Duration::from_secs(10)).await;
assert_eq!(
harness.workers[0].seqs_for("user-1"),
vec![0, 1, 2, 3],
"held messages must deliver in order once a worker returns"
);
harness.stop().await;
}
#[tokio::test]
async fn forced_reap_at_drain_timeout_reroutes_deferred_work() {
let topic = format!("e2e-forced-reap-{}", Uuid::new_v4());
let registry_config = WorkerRegistryConfig {
drain_timeout: Duration::from_millis(400),
..fast_registry_config()
};
let harness = Harness::start(&topic, 1, 2, Duration::from_secs(60), registry_config).await;
let producer = make_producer();
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
// Batch 1 (seq 0–3) reaches its worker and is held in flight there.
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let pinned = sole_arrived_worker(&harness);
let survivor = 1 - pinned;
let pinned_url = harness.workers[pinned].url.clone();
// The pinned worker leaves the pool while its batch is still in flight.
harness.registry.start_draining(&pinned_url);
// Batch 2 (seq 4–7) arrives while batch 1 is held → defers behind the drainer.
for seq in 4..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch 2 to defer", || {
harness.observer.pending_messages() > 0
})
.await;
wait_until(
Duration::from_secs(10),
"the drainer to be force-reaped",
|| {
!harness
.registry
.workers()
.iter()
.any(|w| w.as_ref() == pinned_url)
},
)
.await;
// Release all: batch 1 completes on the reaped worker, batch 2 flushes to the survivor.
guards.clear();
harness.wait_for(8, Duration::from_secs(15)).await;
let reaped = harness.workers[pinned].seqs_for("user-1");
let rerouted = harness.workers[survivor].seqs_for("user-1");
assert!(
!reaped.is_empty(),
"the reaped worker keeps its in-flight batch"
);
assert!(
rerouted.ends_with(&[4, 5, 6, 7]),
"batch 2 arrived after the drain started, so it re-routes: {rerouted:?}"
);
assert_eq!(
[reaped.clone(), rerouted.clone()].concat(),
(0..8).collect::<Vec<_>>(),
"deferred messages re-route to the survivor in order: {reaped:?} then {rerouted:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn consumer_crash_before_commit_redelivers_without_loss() {
let topic = format!("e2e-restart-{}", Uuid::new_v4());
let mut harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Hold both workers so the first batch never acks or commits.
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..6usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
// The worker may record the first delivery after the crash: duplicates allowed, loss not.
harness.crash_consumer();
guards.clear();
harness.restart_consumer(fast_registry_config()).await;
wait_until(
Duration::from_secs(30),
"every message to be redelivered after the crash",
|| {
let delivered: HashSet<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for("user-1"))
.collect();
(0..6).all(|seq| delivered.contains(&seq))
},
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn the_committed_offset_is_the_ledger_frontier() {
let topic = format!("e2e-ledger-commit-{}", Uuid::new_v4());
let mut harness = Harness::start_with_limits(
common::PRODUCTION_PACK_TARGETS,
&topic,
1,
1,
BudgetLimits::new(
NonZeroU64::new(1_000),
NonZeroU64::new(UNBOUNDED_BYTES),
0.8,
)
.expect("a valid budget"),
ChargeLimit::new(NonZeroU64::new(1_000), NonZeroU64::new(UNBOUNDED_BYTES)),
)
.await;
let producer = make_producer();
for seq in 0..10usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
harness.wait_for(10, Duration::from_secs(15)).await;
wait_until(
Duration::from_secs(10),
"the committed offset to reach the frontier",
|| harness.committed_offset(0) == Some(10),
)
.await;
harness.crash_consumer();
harness.restart_consumer(fast_registry_config()).await;
for seq in 10..15usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
harness.wait_for(15, Duration::from_secs(15)).await;
wait_until(
Duration::from_secs(10),
"the committed offset to reach the new frontier",
|| harness.committed_offset(0) == Some(15),
)
.await;
let total: usize = harness.workers.iter().map(|w| w.count()).sum();
assert_eq!(
total, 15,
"a committed frontier behind the accepted work would redeliver here"
);
harness.stop().await;
}
#[tokio::test]
async fn a_held_frontier_stays_uncommitted_until_the_stop_commits_it() {
let metrics = metrics();
let topic = format!("e2e-group-stop-commit-{}", Uuid::new_v4());
let harness = Harness::start_committing_every(
common::PRODUCTION_PACK_TARGETS,
&topic,
1,
1,
Duration::from_secs(3600),
)
.await;
let p0 = TopicPartition::new(&topic, 0);
let producer = make_producer();
produce(&producer, &topic, 0, "tok", "user-a", 0).await;
wait_until(
Duration::from_secs(10),
"the first frontier to commit",
|| harness.committed_offset(0) == Some(1),
)
.await;
for seq in 1..10usize {
produce(&producer, &topic, 0, "tok", "user-a", seq).await;
}
wait_until(Duration::from_secs(10), "every message to settle", || {
harness.workers[0].count() == 10 && harness.ledger.held(&p0).offsets == 0
})
.await;
assert_eq!(
harness.committed_offset(0),
Some(1),
"the interval holds the later frontier"
);
assert_eq!(
uncommitted_offsets(metrics, &topic, 0),
Some(9.0),
"the held frontier is still uncommitted"
);
let group_id = harness.group_id.clone();
harness.stop().await;
wait_until(Duration::from_secs(10), "the stop to commit", || {
broker_committed_offset(&group_id, &topic, 0) == Some(10)
})
.await;
assert_eq!(uncommitted_offsets(metrics, &topic, 0), Some(0.0));
}
// One recorder per test binary; each test reads only its own topic's series.
fn metrics() -> &'static Snapshotter {
static SNAPSHOTTER: OnceLock<Snapshotter> = OnceLock::new();
SNAPSHOTTER.get_or_init(|| {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::set_global_recorder(recorder).expect("no other recorder in the e2e binary");
snapshotter
})
}
fn uncommitted_offsets(metrics: &Snapshotter, topic: &str, partition: i32) -> Option<f64> {
let partition = partition.to_string();
metrics
.snapshot()
.into_vec()
.into_iter()
.find_map(|(key, _, _, value)| {
let key = key.key();
let labeled = |name: &str, expected: &str| {
key.labels()
.any(|label| label.key() == name && label.value() == expected)
};
if key.name() != "kafka_consumer_ledger_uncommitted_offsets"
|| !labeled("topic", topic)
|| !labeled("partition", &partition)
{
return None;
}
match value {
DebugValue::Gauge(value) => Some(value.into_inner()),
_ => None,
}
})
}
const STALL_KEYS: [(i32, &str); 2] = [(0, "user-a"), (1, "user-b")];
async fn stall_two_partitions_on_two_workers(
harness: &Harness,
topic: &str,
) -> Vec<Option<tokio::sync::OwnedMutexGuard<()>>> {
let mut guards = Vec::new();
for worker in &harness.workers {
guards.push(Some(worker.block().await));
}
let producer = make_producer();
let mut sends = Vec::new();
for seq in 0..5usize {
for (partition, key) in STALL_KEYS {
sends.push(produce(&producer, topic, partition, "tok", key, seq));
}
}
futures::future::join_all(sends).await;
wait_until(
Duration::from_secs(10),
"both workers to hold a sub-batch",
|| harness.workers.iter().all(|w| w.arrived_count() > 0),
)
.await;
guards
}
fn accepted_partition(harness: &Harness, idx: usize) -> i32 {
let worker = &harness.workers[idx];
assert_eq!(
worker.count(),
5,
"each worker must hold exactly one key's messages"
);
STALL_KEYS
.iter()
.find(|(_, key)| !worker.seqs_for(key).is_empty())
.map(|(partition, _)| *partition)
.expect("the released worker accepted neither key")
}
#[tokio::test]
async fn a_partition_commits_while_another_stalls() {
let topic = format!("e2e-group-stall-{}", Uuid::new_v4());
// One message per request, or the packer puts both keys on one worker.
let harness = Harness::start_with_pack_targets(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
common::ONE_MESSAGE_PER_REQUEST,
)
.await;
let mut guards = stall_two_partitions_on_two_workers(&harness, &topic).await;
guards[0] = None;
wait_until(
Duration::from_secs(10),
"the released worker to accept its key",
|| harness.workers[0].count() == 5,
)
.await;
let released = accepted_partition(&harness, 0);
let stalled = 1 - released;
wait_until(
Duration::from_secs(10),
"the released partition to commit",
|| harness.committed_offset(released) == Some(5),
)
.await;
assert_eq!(
harness.committed_offset(stalled),
None,
"the stalled send must hold its own partition back"
);
guards[1] = None;
wait_until(
Duration::from_secs(10),
"the stalled partition to commit once released",
|| harness.committed_offset(stalled) == Some(5),
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn a_partition_at_its_budget_pauses_until_a_commit_frees_it() {
let topic = format!("e2e-partition-budget-{}", Uuid::new_v4());
let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)
.expect("a valid budget");
let harness =
Harness::start_with_partition_budget(common::PRODUCTION_PACK_TARGETS, &topic, 1, 1, budget)
.await;
let p0 = TopicPartition::new(&topic, 0);
let guard = harness.workers[0].block().await;
let producer = make_producer();
for seq in 0..5usize {
produce(&producer, &topic, 0, "tok", "user-a", seq).await;
}
wait_until(Duration::from_secs(10), "the budget to fill", || {
harness.ledger.held(&p0).offsets == 5
})
.await;
for seq in 5..20usize {
produce(&producer, &topic, 0, "tok", "user-a", seq).await;
}
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(
harness.ledger.held(&p0).offsets,
5,
"a paused partition fetches nothing more"
);
drop(guard);
wait_until(
Duration::from_secs(10),
"the partition to resume and drain",
|| harness.committed_offset(0) == Some(20),
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn a_partition_at_its_budget_does_not_pause_another() {
let topic = format!("e2e-partition-budget-two-{}", Uuid::new_v4());
let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)
.expect("a valid budget");
// One message per request, or the packer puts both keys on one worker.
let harness =
Harness::start_with_partition_budget(common::ONE_MESSAGE_PER_REQUEST, &topic, 2, 2, budget)
.await;
let mut guards = stall_two_partitions_on_two_workers(&harness, &topic).await;
guards[0] = None;
wait_until(
Duration::from_secs(10),
"the released worker to accept its key",
|| harness.workers[0].count() == 5,
)
.await;
let released = accepted_partition(&harness, 0);
let stalled = 1 - released;
let stalled_partition = TopicPartition::new(&topic, stalled);
let producer = make_producer();
for seq in 5..10usize {
for (partition, key) in STALL_KEYS {
produce(&producer, &topic, partition, "tok", key, seq).await;
}
}
wait_until(
Duration::from_secs(10),
"the released partition to resume and drain",
|| harness.committed_offset(released) == Some(10),
)
.await;
assert_eq!(
harness.ledger.held(&stalled_partition).offsets,
5,
"the partition at its budget fetches nothing more"
);
assert_eq!(harness.committed_offset(stalled), None);
guards[1] = None;
wait_until(
Duration::from_secs(10),
"the stalled partition to resume and drain",
|| harness.committed_offset(stalled) == Some(10),
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn the_in_flight_limit_stops_reading_until_a_commit_frees_room() {
let topic = format!("e2e-in-flight-limit-{}", Uuid::new_v4());
let harness = Harness::start_with_limits(
common::PRODUCTION_PACK_TARGETS,
&topic,
1,
1,
BudgetLimits::new(
NonZeroU64::new(1_000),
NonZeroU64::new(UNBOUNDED_BYTES),
0.8,
)
.expect("a valid budget"),
ChargeLimit::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES)),
)
.await;
let p0 = TopicPartition::new(&topic, 0);
let guard = harness.workers[0].block().await;
let backlog: Vec<(&str, usize)> = (0..120).map(|seq| ("user-a", seq)).collect();
produce_in_one_transaction(&topic, 0, "tok", &backlog).await;
wait_until(
Duration::from_secs(10),
"the in-flight limit to fill",
|| harness.ledger.held(&p0).offsets >= 5,
)
.await;
tokio::time::sleep(Duration::from_secs(1)).await;
assert_eq!(
harness.ledger.held(&p0).offsets,
50,
"a ready backlog stops one poll of the batch size past the limit"
);
drop(guard);
wait_until(
Duration::from_secs(10),
"reading to resume and drain",
|| harness.workers[0].count() == 120,
)
.await;
harness.stop().await;
}
/// On this single-threaded runtime the batcher runs only when the consumer
/// loop yields. A ready backlog keeps `recv` ready on every turn, so the first
/// request must reach the worker before the loop has charged the whole backlog.
#[tokio::test]
async fn a_ready_backlog_does_not_keep_the_batcher_from_running() {
const BACKLOG: usize = 2_000;
let topic = format!("e2e-continuous-backlog-{}", Uuid::new_v4());
let harness = Harness::start_with_limits(
common::PRODUCTION_PACK_TARGETS,
&topic,
1,
1,
BudgetLimits::new(
NonZeroU64::new(100_000),
NonZeroU64::new(UNBOUNDED_BYTES),
0.8,
)
.expect("a valid budget"),
ChargeLimit::new(NonZeroU64::new(100_000), NonZeroU64::new(UNBOUNDED_BYTES)),
)
.await;
let p0 = TopicPartition::new(&topic, 0);
let held_at_arrival: Arc<Mutex<Vec<usize>>> = Arc::new(Mutex::new(Vec::new()));
{
let ledger = Arc::clone(&harness.ledger);
let p0 = p0.clone();
let held_at_arrival = Arc::clone(&held_at_arrival);
harness.workers[0].on_arrival(move || {
held_at_arrival
.lock()
.unwrap()
.push(ledger.held(&p0).offsets);
});
}
let guard = harness.workers[0].block().await;
let backlog: Vec<(&str, usize)> = (0..BACKLOG).map(|seq| ("user-a", seq)).collect();
produce_in_one_transaction(&topic, 0, "tok", &backlog).await;
wait_until(
Duration::from_secs(10),
"the first request to reach the worker",
|| harness.workers[0].arrived_count() >= 1,
)
.await;
let held = held_at_arrival.lock().unwrap()[0];
assert!(
held < BACKLOG / 2,
"the loop charged {held} of {BACKLOG} offsets before the first request reached the worker"
);
drop(guard);
wait_until(Duration::from_secs(10), "the backlog to drain", || {
harness.workers[0].count() == BACKLOG
})
.await;
harness.stop().await;
}
#[tokio::test]
async fn a_partition_paused_at_its_budget_fetches_again_after_a_rebalance_returns_it() {
let topic = format!("e2e-partition-budget-rebalance-{}", Uuid::new_v4());
let budget = BudgetLimits::new(NonZeroU64::new(5), NonZeroU64::new(UNBOUNDED_BYTES), 0.5)
.expect("a valid budget");
let harness = Harness::start_with_partition_budget_committing_every(
common::PRODUCTION_PACK_TARGETS,
&topic,
1,
1,
budget,
Duration::from_secs(20),
)
.await;
let p0 = TopicPartition::new(&topic, 0);
let producer = make_producer();
produce(&producer, &topic, 0, "tok", "user-a", 0).await;
wait_until(
Duration::from_secs(10),
"the first frontier to commit",
|| harness.committed_offset(0) == Some(1),
)
.await;
for seq in 1..6usize {
produce(&producer, &topic, 0, "tok", "user-a", seq).await;
}
wait_until(
Duration::from_secs(10),
"the interval to hold the budget's worth of settled offsets",
|| harness.workers[0].count() == 6 && harness.ledger.held(&p0).offsets == 0,
)
.await;
let generation_before = harness.ledger.generation(&p0);
let second = harness.join_second_consumer().await;
wait_until(
Duration::from_secs(15),
"the rebalance to revoke the partition",
|| harness.ledger.generation(&p0) > generation_before,
)
.await;
assert_eq!(
harness.committed_offset(0),
Some(1),
"the partition must still be at its budget when it is revoked"
);
second.stop().await;
for seq in 6..10usize {
produce(&producer, &topic, 0, "tok", "user-a", seq).await;
}
wait_until(
Duration::from_secs(40),
"the returned partition to fetch again",
|| {
let delivered: HashSet<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for("user-a"))
.collect();
(6..10).all(|seq| delivered.contains(&seq))
},
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn lost_ack_after_processing_replays_without_loss() {
let topic = format!("e2e-lost-ack-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Arm both workers: the batch can land on either.
for w in &harness.workers {
w.ack_lost_once.store(true, Ordering::SeqCst);
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(
Duration::from_secs(15),
"every message delivered and the lost ACK replayed",
|| {
let log = harness.delivery_log.lock().unwrap();
let distinct: HashSet<usize> = log.iter().map(|(_, seq)| *seq).collect();
distinct.len() == 4 && log.len() > 4
},
)
.await;
let log = harness.delivery_log.lock().unwrap().clone();
let seqs: Vec<usize> = log
.iter()
.filter(|(did, _)| did.as_str() == "user-1")
.map(|(_, s)| *s)
.collect();
assert!(
seqs.len() > 4,
"the lost ACK must force a replay (duplicates expected): {seqs:?}"
);
let mut seen = HashSet::new();
let first_deliveries: Vec<usize> = seqs.iter().copied().filter(|s| seen.insert(*s)).collect();
assert_eq!(
first_deliveries,
vec![0, 1, 2, 3],
"no loss, and first deliveries in Kafka order: {seqs:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn poison_batch_fails_safely_without_committing() {
let topic = format!("e2e-poison-{}", Uuid::new_v4());
let mut harness =
Harness::start(&topic, 1, 2, Duration::from_secs(2), fast_registry_config()).await;
let producer = make_producer();
for w in &harness.workers {
w.reject_4xx.store(true, Ordering::SeqCst);
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
assert!(
harness
.wait_for_consumer_exit(Duration::from_secs(20))
.await,
"the consumer must fail the poison batch within the flush timeout"
);
assert_eq!(
harness.workers.iter().map(|w| w.count()).sum::<usize>(),
0,
"a rejected poison batch must not be partially ingested"
);
}
#[tokio::test]
async fn partial_acceptance_fails_the_batch_without_commit() {
let topic = format!("e2e-partial-accept-{}", Uuid::new_v4());
let mut harness =
Harness::start(&topic, 1, 2, Duration::from_secs(2), fast_registry_config()).await;
let producer = make_producer();
for w in &harness.workers {
w.underreport_once.store(true, Ordering::SeqCst);
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
assert!(
harness
.wait_for_consumer_exit(Duration::from_secs(20))
.await,
"an under-acknowledged batch must fail the consumer, not commit"
);
assert_eq!(
harness.committed_offset(0),
None,
"nothing is committed over an under-acknowledged batch"
);
}
#[tokio::test]
async fn drainer_crash_mid_drain_replays_to_survivor_in_order() {
let topic = format!("e2e-drain-crash-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
// Batch 1 (seq 0–3) is held in flight on its worker.
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let drainer = sole_arrived_worker(&harness);
let survivor = 1 - drainer;
// The worker begins draining; batch 2 (seq 4–7) defers behind it.
harness
.registry
.start_draining(&harness.workers[drainer].url);
for seq in 4..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch 2 to defer", || {
harness.observer.pending_messages() > 0
})
.await;
// The drainer crashes mid-drain: its held request fails instead of finishing.
harness.workers[drainer]
.healthy
.store(false, Ordering::SeqCst);
guards.clear();
harness.wait_for(8, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[drainer].count(),
0,
"the crashed drainer ingested nothing"
);
assert_eq!(
harness.workers[survivor].seqs_for("user-1"),
(0..8).collect::<Vec<_>>(),
"all work replays to the survivor in order"
);
harness.stop().await;
}
#[rstest]
#[case::production_targets(common::PRODUCTION_PACK_TARGETS)]
#[case::one_message_per_request(common::ONE_MESSAGE_PER_REQUEST)]
#[tokio::test]
async fn full_pool_send_failure_with_multiple_keys_recovers_in_order(
#[case] pack_targets: PackTargets,
) {
let topic = format!("e2e-pool-loss-{}", Uuid::new_v4());
let harness = Harness::start_with_pack_targets(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
pack_targets,
)
.await;
let producer = make_producer();
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
// One message per request puts each key on its own worker; production targets pack both.
let split = pack_targets == common::ONE_MESSAGE_PER_REQUEST;
wait_until(Duration::from_secs(10), "work to be in flight", || {
if split {
harness.workers.iter().all(|w| w.arrived_count() > 0)
} else {
harness.workers.iter().any(|w| w.arrived_count() > 0)
}
})
.await;
for w in &harness.workers {
w.healthy.store(false, Ordering::SeqCst);
}
guards.clear();
wait_until(
Duration::from_secs(10),
"all failed work to be stashed",
|| harness.observer.pending_messages() == 8,
)
.await;
harness.workers[1].healthy.store(true, Ordering::SeqCst);
harness.wait_for(8, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[0].count(),
0,
"the still-dead worker recorded nothing"
);
for user in ["user-1", "user-2"] {
assert_eq!(
harness.workers[1].seqs_for(user),
(0..4).collect::<Vec<_>>(),
"{user} must drain to the recovered worker in order"
);
}
harness.stop().await;
}
#[tokio::test]
async fn headerless_messages_survive_send_failure_replay() {
let topic = format!("e2e-headerless-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Take worker 1 out so the batch lands on worker 0 as one sub-batch.
harness.workers[1].healthy.store(false, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to leave the pool",
|| !in_pool(&harness, &harness.workers[1].url),
)
.await;
let guard0 = harness.workers[0].block().await;
for seq in 0..3usize {
produce_raw(
&producer,
&topic,
0,
&format!("k{seq}"),
format!(r#"{{"seq":{seq}}}"#).as_bytes(),
None,
)
.await;
}
wait_until(Duration::from_secs(10), "batch to reach worker 0", || {
harness.workers[0].arrived_count() > 0
})
.await;
harness.workers[0].healthy.store(false, Ordering::SeqCst);
harness.workers[1].healthy.store(true, Ordering::SeqCst);
wait_until(
Duration::from_secs(10),
"worker 1 to rejoin the pool",
|| in_pool(&harness, &harness.workers[1].url),
)
.await;
drop(guard0);
harness.wait_for(3, Duration::from_secs(15)).await;
assert_eq!(
harness.workers[0].count(),
0,
"failed worker recorded nothing"
);
// Headerless messages have an empty distinct_id and distinct keys, so order is free.
let mut seqs = harness.workers[1].seqs_for("");
seqs.sort_unstable();
assert_eq!(
seqs,
vec![0, 1, 2],
"headerless messages must replay exactly once"
);
harness.stop().await;
}
#[tokio::test]
async fn non_utf8_payload_is_delivered_not_lost() {
let topic = format!("e2e-non-utf8-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
1,
1,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
let headers = OwnedHeaders::new()
.insert(Header {
key: "token",
value: Some("tok"),
})
.insert(Header {
key: "distinct_id",
value: Some("user-1"),
});
produce_raw(
&producer,
&topic,
0,
"k",
&[0xF0, 0x28, 0x8C, 0x28],
Some(headers),
)
.await;
harness.wait_for(1, Duration::from_secs(10)).await;
assert_eq!(
harness.workers[0].seqs_for("user-1"),
vec![0],
"the event must arrive (with its body nulled) rather than be dropped"
);
harness.stop().await;
}
#[tokio::test]
async fn fleet_redeploy_migrates_work_to_new_workers_without_loss() {
let topic = format!("e2e-redeploy-{}", Uuid::new_v4());
let mut harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Steady traffic on two keys lands on the old fleet.
for seq in 0..10usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
harness.wait_for(20, Duration::from_secs(15)).await;
// The fleet is replaced wholesale.
let new_a = FakeWorker::start_logged(Arc::clone(&harness.delivery_log)).await;
let new_b = FakeWorker::start_logged(Arc::clone(&harness.delivery_log)).await;
let desired: HashSet<WorkerId> = [new_a.url.as_str(), new_b.url.as_str()]
.into_iter()
.map(WorkerId::from)
.collect();
reconcile_membership(&harness.registry, &desired);
let old_count = harness.workers.len();
harness.workers.push(new_a);
harness.workers.push(new_b);
// Traffic continues through the switchover.
for seq in 10..20usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
harness.wait_for(40, Duration::from_secs(20)).await;
wait_until(Duration::from_secs(5), "old workers to be reaped", || {
harness.registry.worker_count() == 2
})
.await;
let log = harness.delivery_log.lock().unwrap().clone();
assert_eq!(log.len(), 40, "exactly-once across the redeploy");
for user in ["user-1", "user-2"] {
let seqs: Vec<usize> = log
.iter()
.filter(|(d, _)| d.as_str() == user)
.map(|(_, s)| *s)
.collect();
assert_eq!(
seqs,
(0..20).collect::<Vec<_>>(),
"{user} must stay in order across the redeploy: {seqs:?}"
);
}
let new_fleet_received: usize = harness.workers[old_count..].iter().map(|w| w.count()).sum();
assert!(
new_fleet_received > 0,
"the new fleet must take over the traffic"
);
harness.stop().await;
}
#[tokio::test]
async fn second_consumer_joining_the_group_preserves_all_messages() {
let topic = format!("e2e-two-consumers-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
// Hold an uncommitted batch while the first consumer owns both partitions.
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..5usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(Duration::from_secs(10), "work to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let second = harness.join_second_consumer().await;
// Release the batch so the first consumer polls again and the rebalance completes.
guards.clear();
tokio::time::sleep(Duration::from_secs(2)).await;
for seq in 5..10usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(
Duration::from_secs(30),
"every message to be delivered at least once across the rebalance",
|| {
["user-1", "user-2"].iter().all(|user| {
let delivered: HashSet<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for(user))
.collect();
(0..10).all(|s| delivered.contains(&s))
})
},
)
.await;
second.stop().await;
harness.stop().await;
}
#[rstest]
#[case::eager(RebalanceMode::Eager)]
#[case::cooperative(RebalanceMode::Cooperative)]
#[tokio::test]
async fn a_drain_commits_the_in_flight_messages_before_the_new_owner_reads(
#[case] rebalance_mode: RebalanceMode,
) {
let topic = format!("e2e-revoke-drain-{}", Uuid::new_v4());
let harness =
Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)
.await;
let producer = make_producer();
let mut guards = Vec::new();
for w in &harness.workers {
guards.push(w.block().await);
}
for seq in 0..5usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(Duration::from_secs(10), "work to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let second = harness.join_second_consumer().await;
// The drain waits on the gated requests.
tokio::time::sleep(Duration::from_secs(2)).await;
guards.clear();
for seq in 5..10usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
let expected: HashSet<(String, usize)> = ["user-1", "user-2"]
.iter()
.flat_map(|user| (0..10usize).map(|seq| (user.to_string(), seq)))
.collect();
wait_until(
Duration::from_secs(30),
"every message to be delivered",
|| {
let delivered: HashSet<(String, usize)> = harness
.delivery_log
.lock()
.unwrap()
.iter()
.cloned()
.collect();
expected.is_subset(&delivered)
},
)
.await;
// Time for a replay by the new owner to arrive.
tokio::time::sleep(Duration::from_secs(2)).await;
let log = harness.delivery_log.lock().unwrap().clone();
let mut seen = HashSet::new();
let duplicates: Vec<&(String, usize)> =
log.iter().filter(|entry| !seen.insert(*entry)).collect();
assert!(
duplicates.is_empty(),
"the new owner processed again what the old owner finished: {duplicates:?}"
);
second.stop().await;
harness.stop().await;
}
fn processed_counts(harness: &Harness) -> HashMap<(String, usize), usize> {
let mut counts = HashMap::new();
for entry in harness.delivery_log.lock().unwrap().iter() {
*counts.entry(entry.clone()).or_insert(0) += 1;
}
counts
}
#[rstest]
#[case::eager(RebalanceMode::Eager)]
#[case::cooperative(RebalanceMode::Cooperative)]
#[tokio::test]
async fn a_drain_holds_every_partition_and_dispatches_nothing_until_its_release(
#[case] rebalance_mode: RebalanceMode,
) {
let topic = format!("e2e-drain-hold-{}", Uuid::new_v4());
let harness =
Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(20), rebalance_mode)
.await;
let producer = make_producer();
let mut guards = Vec::new();
for w in &harness.workers {
guards.push(w.block().await);
}
for seq in 0..5usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(
Duration::from_secs(10),
"every produced message to dispatch",
|| harness.poll_sizes().iter().sum::<usize>() == 10,
)
.await;
let second = harness.join_second_consumer().await;
wait_for_group(
&harness.group_id,
Duration::from_secs(10),
"hold a revoke",
|group| group.holds_a_revoke(rebalance_mode),
)
.await;
// The revoke arrives on the next heartbeat; the gated requests hold the drain.
tokio::time::sleep(Duration::from_secs(2)).await;
let dispatched = harness.poll_sizes().len();
for seq in 5..15usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
tokio::time::sleep(Duration::from_secs(2)).await;
assert_eq!(
harness.poll_sizes().len(),
dispatched,
"librdkafka pauses the whole assignment until the unassign, so a drain dispatches no poll"
);
let group = group_state(&harness.group_id);
assert!(
group
.as_ref()
.is_some_and(|group| group.holds_a_revoke(rebalance_mode)),
"the drain still holds the revoke: {group:?}"
);
guards.clear();
let expected: HashSet<(String, usize)> = ["user-1", "user-2"]
.iter()
.flat_map(|user| (0..15usize).map(|seq| (user.to_string(), seq)))
.collect();
wait_until(
Duration::from_secs(30),
"every message to be delivered",
|| {
let delivered: HashSet<(String, usize)> = harness
.delivery_log
.lock()
.unwrap()
.iter()
.cloned()
.collect();
expected.is_subset(&delivered)
},
)
.await;
wait_for_group(
&harness.group_id,
Duration::from_secs(10),
"hand the partition over",
GroupState::handed_over,
)
.await;
// Time for a replay by either owner to arrive.
tokio::time::sleep(Duration::from_secs(2)).await;
let duplicates: Vec<((String, usize), usize)> = processed_counts(&harness)
.into_iter()
.filter(|(_, count)| *count > 1)
.collect();
assert!(
duplicates.is_empty(),
"processed more than once: {duplicates:?}"
);
second.stop().await;
harness.stop().await;
}
#[rstest]
#[case::eager(RebalanceMode::Eager)]
#[case::cooperative(RebalanceMode::Cooperative)]
#[tokio::test]
async fn a_drain_that_times_out_hands_the_partitions_over_uncommitted(
#[case] rebalance_mode: RebalanceMode,
) {
let topic = format!("e2e-drain-timeout-{}", Uuid::new_v4());
let harness =
Harness::start_draining_revokes(&topic, 2, 2, Duration::from_secs(3), rebalance_mode).await;
let producer = make_producer();
let partitions = [
TopicPartition::new(topic.clone(), 0),
TopicPartition::new(topic.clone(), 1),
];
let users = [("user-1", 0), ("user-2", 1)];
let mut guards = Vec::new();
for w in &harness.workers {
guards.push(w.block().await);
}
for seq in 0..5usize {
for (user, partition) in users {
produce(&producer, &topic, partition, "tok", user, seq).await;
}
}
wait_until(
Duration::from_secs(10),
"every produced message to dispatch",
|| harness.poll_sizes().iter().sum::<usize>() == 10,
)
.await;
let generations_before: Vec<u64> = partitions
.iter()
.map(|partition| harness.ledger.generation(partition))
.collect();
let bumps = |harness: &Harness| -> Vec<u64> {
let mut bumps: Vec<u64> = partitions
.iter()
.zip(&generations_before)
.map(|(partition, before)| harness.ledger.generation(partition) - before)
.collect();
bumps.sort_unstable();
bumps
};
let joined = tokio::time::Instant::now();
let second = harness.join_second_consumer().await;
// The gated requests outlive the drain timeout.
wait_for_group(
&harness.group_id,
Duration::from_secs(20),
"hand the partition over",
GroupState::handed_over,
)
.await;
assert!(
joined.elapsed() >= Duration::from_secs(3),
"the drain held the revoke for its timeout"
);
assert_eq!(harness.committed_offset(0), None);
assert_eq!(harness.committed_offset(1), None);
// The release bumps the generation of each revoked partition, and an
// assignment bumps the generation of each partition it returns.
let expected_bumps = match rebalance_mode {
RebalanceMode::Eager => vec![1, 2],
RebalanceMode::Cooperative => vec![0, 1],
};
wait_until(
Duration::from_secs(10),
"the first consumer to apply its new assignment",
|| bumps(&harness) == expected_bumps,
)
.await;
let moved = partitions
.iter()
.zip(&generations_before)
.find(|(partition, before)| harness.ledger.generation(partition) == *before + 1)
.map(|(partition, _)| partition.partition)
.expect("one partition left the first consumer");
guards.clear();
let read_again = |partition: i32| partition == moved || rebalance_mode == RebalanceMode::Eager;
let processed_as_expected = |harness: &Harness| {
let counts = processed_counts(harness);
users.iter().all(|(user, partition)| {
let expected = if read_again(*partition) { 2 } else { 1 };
(0..5usize).all(|seq| counts.get(&(user.to_string(), seq)) == Some(&expected))
})
};
wait_until(
Duration::from_secs(30),
"the released requests and the new owners to process every message",
|| processed_as_expected(&harness),
)
.await;
wait_until(
Duration::from_secs(10),
"each owner to commit its partition",
|| harness.committed_offset(0) == Some(5) && harness.committed_offset(1) == Some(5),
)
.await;
// Time for a replay by either owner to arrive.
tokio::time::sleep(Duration::from_secs(2)).await;
assert!(
processed_as_expected(&harness),
"the old owner's released requests process every message once, and each new assignment processes its partition once more: {:?}",
processed_counts(&harness)
);
assert_eq!(
bumps(&harness),
expected_bumps,
"no later rebalance touched the first consumer's partitions"
);
let commits_of_moved: Vec<i64> = harness
.debug_recorder
.backlog()
.into_iter()
.filter_map(|event| match event.kind {
DebugEventKind::CommitSubmitted { offsets } => Some(offsets),
_ => None,
})
.flatten()
.filter(|offset| offset.partition == moved)
.map(|offset| offset.offset)
.collect();
assert_eq!(
commits_of_moved,
Vec::<i64>::new(),
"the old owner's late completions commit nothing for the partition it handed over"
);
second.stop().await;
harness.stop().await;
}
#[tokio::test]
async fn partition_lost_and_regained_keeps_the_consumer_alive() {
let topic = format!("e2e-regain-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
let partitions = [
TopicPartition::new(topic.clone(), 0),
TopicPartition::new(topic.clone(), 1),
];
// Hold the first batch in flight while the group changes underneath it.
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..5usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(Duration::from_secs(10), "work to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let generations_before: Vec<u64> = partitions
.iter()
.map(|partition| harness.ledger.generation(partition))
.collect();
// A second consumer joins and takes a partition away.
let second = harness.join_second_consumer().await;
// A trickle keeps the consumer polling until the rebalance reaches its ledger.
let rebalanced = || {
partitions
.iter()
.zip(&generations_before)
.any(|(partition, before)| harness.ledger.generation(partition) > *before)
};
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
let mut trickle_seq = 1000usize;
while !rebalanced() {
assert!(
tokio::time::Instant::now() < deadline,
"timed out waiting for the rebalance to reach the first consumer's ledger"
);
produce(&producer, &topic, 0, "tok", "user-1", trickle_seq).await;
trickle_seq += 1;
tokio::time::sleep(Duration::from_millis(20)).await;
}
// The released offsets carry replaced generations, so they settle as stale.
guards.clear();
for seq in 5..10usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(
Duration::from_secs(30),
"every message to be delivered across the join",
|| {
["user-1", "user-2"].iter().all(|user| {
let delivered: HashSet<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for(user))
.collect();
(0..10).all(|s| delivered.contains(&s))
})
},
)
.await;
// The partition returns under a new generation.
second.stop().await;
for seq in 10..15usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
wait_until(
Duration::from_secs(30),
"the returned partition to consume under its new assignment",
|| {
["user-1", "user-2"].iter().all(|user| {
let delivered: HashSet<usize> = harness
.workers
.iter()
.flat_map(|w| w.seqs_for(user))
.collect();
(10..15).all(|s| delivered.contains(&s))
})
},
)
.await;
let epochs = harness.dispatched_assignment_epochs();
assert!(
epochs.last() > epochs.first(),
"polls after the regain go out under a later assignment epoch: {epochs:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn fenced_static_member_exits_on_fatal_error() {
let topic = format!("e2e-fenced-{}", Uuid::new_v4());
create_topic(&topic, 1).await;
let worker = FakeWorker::start().await;
let urls = vec![worker.url.clone()];
let registry = Arc::new(WorkerRegistry::new(&urls, fast_registry_config()));
let probe = CancellationToken::new();
Arc::clone(®istry).start_probing(probe.clone());
let pool_source = WorkerPoolSource::new(Arc::clone(®istry), RoutingStrategy::default());
let transport = Arc::new(test_transport());
let mut manager = Manager::builder("e2e-fenced")
.with_trap_signals(false)
.build();
let handle = manager.register("consumer", ComponentOptions::new());
let group = format!("e2e-{}", Uuid::new_v4());
let (batcher, outputs) = common::batcher(
pool_source,
Arc::clone(&transport),
Duration::from_secs(60),
Duration::from_millis(200),
common::PRODUCTION_PACK_TARGETS,
None,
);
let consumer = IngestionConsumer::from_parts(
make_kafka_consumer(&topic, &group, Some("pod-1")),
batcher,
outputs,
transport,
urls,
IngestionConsumerOptions {
batch_size: 50,
batch_size_bytes: 0,
batch_timeout: Duration::from_millis(100),
group_id: "e2e-test".to_string(),
debug_recorder: None,
commit_interval: Duration::ZERO,
ledger: Arc::new(TopicOffsetLedger::new()),
partition_budget: BudgetLimits::unset(),
in_flight_limit: ChargeLimit::unset(),
revoke_drain_timeout: None,
},
handle,
);
let task = tokio::spawn(async move { consumer.process().await });
// Prove the first instance is consuming before it gets fenced.
let producer = make_producer();
produce(&producer, &topic, 0, "tok", "user-1", 0).await;
wait_until(Duration::from_secs(10), "first instance to consume", || {
worker.count() >= 1
})
.await;
// The usurper joins with the SAME instance id; polling drives the join.
let usurper = make_kafka_consumer(&topic, &group, Some("pod-1"));
let usurper_task = tokio::spawn(async move {
let mut stream = usurper.stream();
// Errors are items too; read until the timeout so the usurper joins.
let _ = tokio::time::timeout(Duration::from_secs(30), async {
while stream.next().await.is_some() {}
})
.await;
});
let exited = tokio::time::timeout(Duration::from_secs(30), task)
.await
.is_ok();
assert!(
exited,
"the fenced instance must exit with a fatal error, not keep polling a dead client"
);
usurper_task.abort();
probe.cancel();
}
#[derive(Clone, Copy, Debug)]
enum Churn {
Drain,
Fail,
Mixed,
}
async fn sleep_or_cancel(token: &CancellationToken, ms: u64) -> bool {
tokio::select! {
_ = token.cancelled() => true,
_ = tokio::time::sleep(Duration::from_millis(ms)) => false,
}
}
/// Churn one worker at a time until cancelled, then restore all.
fn spawn_churn(
harness: &Harness,
churn: Churn,
token: CancellationToken,
) -> tokio::task::JoinHandle<()> {
let registry = Arc::clone(&harness.registry);
let controls: Vec<(String, Arc<AtomicBool>)> = harness
.workers
.iter()
.map(|w| (w.url.clone(), Arc::clone(&w.ingest_ok)))
.collect();
tokio::spawn(async move {
let mut i = 0usize;
while !token.is_cancelled() {
let (url, ingest_ok) = &controls[i % controls.len()];
let drain = match churn {
Churn::Drain => true,
Churn::Fail => false,
Churn::Mixed => i.is_multiple_of(2),
};
if drain {
registry.start_draining(url);
if sleep_or_cancel(&token, 120).await {
break;
}
registry.add_worker(WorkerId::from(url.as_str()));
} else {
ingest_ok.store(false, Ordering::SeqCst);
if sleep_or_cancel(&token, 120).await {
break;
}
ingest_ok.store(true, Ordering::SeqCst);
}
// A gap with everyone healthy so the backlog can drain.
if sleep_or_cancel(&token, 80).await {
break;
}
i += 1;
}
for (url, ingest_ok) in &controls {
ingest_ok.store(true, Ordering::SeqCst);
registry.add_worker(WorkerId::from(url.as_str()));
}
})
}
async fn run_churn_suite(churn: Churn, strict_order: bool) {
let topic = format!("e2e-churn-{churn:?}-{}", Uuid::new_v4());
let dids = 8usize;
let per_did = 60usize;
let total = dids * per_did;
let partitions = 4i32;
let harness = Harness::start(
&topic,
partitions,
4,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
let churn_token = CancellationToken::new();
let churn_handle = spawn_churn(&harness, churn, churn_token.clone());
// Interleave distinct_ids so each Kafka batch mixes many keys.
for seq in 0..per_did {
for d in 0..dids {
produce(
&producer,
&topic,
(d as i32) % partitions,
"tok",
&format!("user-{d}"),
seq,
)
.await;
}
}
harness.wait_for(total, Duration::from_secs(90)).await;
// Quiesce churn so the final assertions see a settled log.
churn_token.cancel();
let _ = churn_handle.await;
let log = harness.delivery_log.lock().unwrap().clone();
assert_eq!(
log.len(),
total,
"exactly-once: expected {total} deliveries, got {}",
log.len()
);
let expected: Vec<usize> = (0..per_did).collect();
for d in 0..dids {
let did = format!("user-{d}");
let seqs: Vec<usize> = log
.iter()
.filter(|(id, _)| *id == did)
.map(|(_, s)| *s)
.collect();
let mut sorted = seqs.clone();
sorted.sort_unstable();
assert_eq!(
sorted, expected,
"{did}: messages lost or duplicated under {churn:?} churn: {seqs:?}"
);
if strict_order {
assert_eq!(
seqs, expected,
"{did}: messages delivered out of order under {churn:?} churn: {seqs:?}"
);
}
}
// Leaked bookkeeping would skew placement or strand messages.
wait_until(
Duration::from_secs(10),
"batcher pending/in-flight to drain to zero after churn",
|| harness.observer.pending_messages() == 0 && harness.observer.total_in_flight() == 0,
)
.await;
harness.stop().await;
}
#[tokio::test]
async fn churn_drain_preserves_order() {
run_churn_suite(Churn::Drain, true).await;
}
#[tokio::test]
async fn churn_send_failures_are_exactly_once() {
run_churn_suite(Churn::Fail, false).await;
}
#[tokio::test]
async fn churn_mixed_drain_and_failures_are_exactly_once() {
run_churn_suite(Churn::Mixed, false).await;
}
#[tokio::test]
async fn flapping_worker_does_not_pin_batch_past_flush_timeout() {
let topic = format!("e2e-flap-{}", Uuid::new_v4());
// Short flush timeout so the bound is observable quickly; concurrency > 1.
let mut harness =
Harness::start(&topic, 1, 2, Duration::from_secs(2), fast_registry_config()).await;
let producer = make_producer();
let mut guards: Vec<Option<tokio::sync::OwnedMutexGuard<()>>> = Vec::new();
for w in &harness.workers {
guards.push(Some(w.block().await));
}
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
wait_until(Duration::from_secs(10), "batch to reach a worker", || {
harness.workers.iter().any(|w| w.arrived_count() > 0)
})
.await;
let target = sole_arrived_worker(&harness);
let other = 1 - target;
// target stays ready but nacks; other leaves the pool, so the flush has nowhere else to go.
harness.workers[target]
.ingest_ok
.store(false, Ordering::SeqCst);
harness.workers[other]
.healthy
.store(false, Ordering::SeqCst);
guards[other] = None;
guards[target] = None;
assert!(
harness
.wait_for_consumer_exit(Duration::from_secs(20))
.await,
"consumer must fail the batch within the flush timeout, not spin forever"
);
assert_eq!(
harness.workers.iter().map(|w| w.count()).sum::<usize>(),
0,
"nothing should be delivered (target only errors; other is down)"
);
}
#[tokio::test]
async fn nacking_worker_fences_worker_stream_and_reroutes_in_order() {
let topic = format!("e2e-fence-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
2,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
for seq in 0..4usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
produce(&producer, &topic, 1, "tok", "user-2", seq).await;
}
harness.wait_for(8, Duration::from_secs(10)).await;
// Take down user-1's worker: it nacks and leaves the pool.
let dead_idx = if !harness.workers[0].seqs_for("user-1").is_empty() {
0
} else {
1
};
let live_idx = 1 - dead_idx;
harness.workers[dead_idx]
.healthy
.store(false, Ordering::Relaxed);
tokio::time::sleep(Duration::from_millis(600)).await;
for seq in 4..8usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
harness.wait_for(12, Duration::from_secs(15)).await;
let live_seqs = harness.workers[live_idx].seqs_for("user-1");
let batch2_seqs: Vec<usize> = live_seqs.iter().copied().filter(|&s| s >= 4).collect();
assert_eq!(
batch2_seqs,
vec![4, 5, 6, 7],
"post-fence messages for user-1 missing or out-of-order on the live worker: {live_seqs:?}"
);
harness.stop().await;
}
#[tokio::test]
async fn same_key_across_partitions_completes_and_commits() {
let topic = format!("e2e-cross-partition-key-{}", Uuid::new_v4());
let harness = Harness::start(
&topic,
2,
1,
Duration::from_secs(60),
fast_registry_config(),
)
.await;
let producer = make_producer();
for seq in 0..3usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
for seq in 3..6usize {
produce(&producer, &topic, 1, "tok", "user-1", seq).await;
}
harness.wait_for(6, Duration::from_secs(15)).await;
for seq in 6..9usize {
produce(&producer, &topic, 0, "tok", "user-1", seq).await;
}
for seq in 9..12usize {
produce(&producer, &topic, 1, "tok", "user-1", seq).await;
}
harness.wait_for(12, Duration::from_secs(15)).await;
let seqs = harness.workers[0].seqs_for("user-1");
let mut sorted = seqs.clone();
sorted.sort_unstable();
assert_eq!(
sorted,
(0..12).collect::<Vec<_>>(),
"every message must land exactly once; got {seqs:?}"
);
let partition_0: Vec<usize> = seqs
.iter()
.copied()
.filter(|s| *s < 3 || (6..9).contains(s))
.collect();
let partition_1: Vec<usize> = seqs
.iter()
.copied()
.filter(|s| (3..6).contains(s) || *s >= 9)
.collect();
assert_eq!(
partition_0,
vec![0, 1, 2, 6, 7, 8],
"partition 0 must keep offset order"
);
assert_eq!(
partition_1,
vec![3, 4, 5, 9, 10, 11],
"partition 1 must keep offset order"
);
harness.stop().await;
}