Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ingestion-consumer): remove the transport per-worker cap (#121245)

José Sequeira25027f9ed1c3
diff --git a/rust/ingestion-consumer/src/config.rs b/rust/ingestion-consumer/src/config.rs
index 5639e13547bb..de6d02bb5092 100644
--- a/rust/ingestion-consumer/src/config.rs
+++ b/rust/ingestion-consumer/src/config.rs
@@ -326,10 +326,11 @@ pub struct Config {
#[envconfig(default = "http://localhost:9001")]
pub worker_addresses: String,
- /// Cap on un-acked sub-batches per worker stream. Ideally aligned with the
- /// worker's `BatchingPipeline.concurrentBatches`
- /// (`INGESTION_WORKER_CONCURRENT_BATCHES` on the Node.js side) so the
- /// stream backpressures before the worker stalls its reads.
+ /// The most requests the batcher keeps in flight on one worker, and the
+ /// governor's permits per worker at its ceiling. Aligned with the worker's
+ /// `BatchingPipeline.concurrentBatches` (`INGESTION_WORKER_CONCURRENT_BATCHES`
+ /// on the Node.js side), because the worker queues requests past it
+ /// without saying so.
#[envconfig(from = "INGESTION_WORKER_CONCURRENT_BATCHES", default = "1")]
pub ingestion_worker_concurrent_batches: usize,
@@ -512,9 +513,8 @@ impl Config {
format!("{}:{}", self.bind_host, self.bind_port)
}
- /// The batcher state machine. It reuses the stream's un-acked cap as its
- /// per-worker request cap, the parked-retry interval for every retry, and
- /// the deferred-flush timeout as its stall timeout.
+ /// The batcher state machine. It takes the parked-retry interval for every
+ /// retry and the deferred-flush timeout as its stall timeout.
pub fn batcher_state_machine(&self) -> Result<BatcherStateMachine, String> {
let mut packer = Packer::new(PackTargets {
events: NonZeroUsize::new(self.pack_target_events),
diff --git a/rust/ingestion-consumer/src/grpc_transport.rs b/rust/ingestion-consumer/src/grpc_transport.rs
index fc96fb878eb5..b09f690bfc3b 100644
--- a/rust/ingestion-consumer/src/grpc_transport.rs
+++ b/rust/ingestion-consumer/src/grpc_transport.rs
@@ -31,11 +31,10 @@
//! Nothing for a fenced key can leapfrog the failure, because everything for
//! it was either in the ledger, the queue, or fenced on arrival.
//!
-//! Backpressure: the queue is unbounded (so enqueue stays synchronous and
-//! ordered) but the worker stream keeps at most `max_unacked` sub-batches un-acked on
-//! the stream; queue depth is bounded in practice by the consumer's
-//! `max_in_flight_batches`, whose batches cannot complete until their sends
-//! resolve.
+//! The queue is unbounded, so enqueue stays synchronous and ordered. The
+//! batcher bounds its depth: it keeps at most `INGESTION_WORKER_CONCURRENT_BATCHES`
+//! requests in flight per worker and sends nothing more until one settles,
+//! so every queued sub-batch goes on the wire at once.
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
@@ -194,9 +193,6 @@ pub struct GrpcTransport {
consumer_id: String,
/// How each worker's stream address is derived from its HTTP URL.
grpc_port: GrpcPort,
- /// Max un-acked sub-batches per worker stream (aligned with the worker's
- /// `concurrentBatches`).
- max_unacked: usize,
/// Fence the worker stream when un-acked work sees no ack for this long.
ack_timeout: Duration,
/// Bumped on Kafka partition assignment; stamped on every sub-batch so
@@ -211,13 +207,11 @@ pub struct GrpcTransport {
}
impl GrpcTransport {
- pub fn new(grpc_port: GrpcPort, max_unacked: usize, ack_timeout: Duration) -> Self {
- assert!(max_unacked > 0, "max_unacked must be > 0");
+ pub fn new(grpc_port: GrpcPort, ack_timeout: Duration) -> Self {
Self {
worker_streams: DashMap::new(),
consumer_id: make_consumer_id(),
grpc_port,
- max_unacked,
ack_timeout,
max_body_bytes: DEFAULT_MAX_BODY_BYTES,
// A fresh epoch for standalone construction; production wiring
@@ -237,11 +231,6 @@ impl GrpcTransport {
self.assignment_epoch = epoch;
}
- /// The most sub-batches one worker stream keeps un-acked.
- pub fn max_unacked(&self) -> usize {
- self.max_unacked
- }
-
/// The assignment epoch this transport stamps on sub-batches.
pub fn assignment_epoch(&self) -> AssignmentEpoch {
self.assignment_epoch.clone()
@@ -345,7 +334,6 @@ impl GrpcTransport {
worker_url: Arc::from(worker_url),
grpc_url: grpc_url(worker_url, self.grpc_port),
consumer_id: self.consumer_id.clone(),
- max_unacked: self.max_unacked,
ack_timeout: self.ack_timeout,
assignment_epoch: self.assignment_epoch.clone(),
};
@@ -354,10 +342,12 @@ impl GrpcTransport {
}
/// Tear down a departed worker's stream. Dropping it closes its queue
- /// sender, so the runner fences whatever is still in flight **in order**,
- /// with the messages intact, and exits on its own. Aborting the task would
- /// instead drop the un-acked sends unresolved, which the caller can only
- /// recover by crashing and replaying — so let the graceful path run.
+ /// sender: the runner takes no more work, lets the sub-batches already on
+ /// the wire settle by ack or by the ack watchdog, and exits on its own.
+ /// A reaped worker that is slow rather than dead so finishes its request
+ /// instead of a survivor repeating it while it still runs. Aborting the
+ /// task would instead drop the un-acked sends unresolved, which the caller
+ /// can only recover by crashing and replaying.
pub fn remove_worker(&self, worker_url: &str) {
self.worker_streams.remove(worker_url);
}
@@ -410,7 +400,6 @@ struct WorkerStreamRunner {
worker_url: Arc<str>,
grpc_url: String,
consumer_id: String,
- max_unacked: usize,
ack_timeout: Duration,
assignment_epoch: AssignmentEpoch,
}
@@ -582,8 +571,9 @@ impl WorkerStreamRunner {
}
}
- /// Send the held first item, then pull more only while the ledger has
- /// room — the worker-aligned concurrency cap. `Err` ends the stream.
+ /// The batcher caps the requests in flight per worker, so nothing is held
+ /// back here. A closed queue stops the filling; the ledger still drains by
+ /// ack. `Err` ends the stream.
fn fill_ledger(
&self,
out_tx: &mpsc::UnboundedSender<IngestStreamRequest>,
@@ -592,19 +582,17 @@ impl WorkerStreamRunner {
ledger: &mut VecDeque<LedgerEntry>,
next_seq: &mut u64,
) -> Result<(), StreamEnd> {
- while ledger.len() < self.max_unacked {
+ loop {
let item = match pending_first.take() {
Some(item) => Some(item),
None => match queue.try_recv() {
Ok(item) => Some(item),
Err(mpsc::error::TryRecvError::Empty) => None,
Err(mpsc::error::TryRecvError::Disconnected) => {
- return Err(if ledger.is_empty() {
- StreamEnd::QueueClosed
- } else {
- // Resolve the tail before exiting.
- self.fence(None, ledger, queue, "queue closed")
- });
+ if ledger.is_empty() {
+ return Err(StreamEnd::QueueClosed);
+ }
+ None
}
},
};
@@ -654,8 +642,9 @@ impl WorkerStreamRunner {
.record(entry.sent_at.elapsed().as_secs_f64());
}
- /// Wait for an ack, or for new work when the ledger has room. `Ok(None)`
- /// means new work was held in `pending_first`; `Err` ends the stream.
+ /// Wait for an ack, or for new work while the queue is open. `Ok(None)`
+ /// means new work was held in `pending_first`, or the queue closed under
+ /// the wait and the ledger still drains; `Err` ends the stream.
///
/// The ack-progress watchdog bounds how long the oldest un-acked
/// sub-batch may sit unacked. Each entry carries its own deadline, armed
@@ -692,12 +681,9 @@ impl WorkerStreamRunner {
);
Err(self.fence(pending_first.take(), ledger, queue, "ack progress timeout"))
}
- item = queue.recv(), if ledger.len() < self.max_unacked => match item {
- Some(item) => {
- *pending_first = Some(item);
- Ok(None)
- }
- None => Err(self.fence(None, ledger, queue, "queue closed")),
+ item = queue.recv(), if !queue.is_closed() => {
+ *pending_first = item;
+ Ok(None)
}
}
}
@@ -754,7 +740,6 @@ impl WorkerStreamRunner {
);
return Err(self.fence_with(true, None, ledger, queue, "worker busy"));
}
- let was_full = ledger.len() >= self.max_unacked;
let Some(index) = ledger
.iter()
.position(|e| e.seq == response.seq && e.acked.is_none())
@@ -801,13 +786,6 @@ impl WorkerStreamRunner {
messages,
}));
}
- if was_full && ledger.len() < self.max_unacked && !queue.is_empty() {
- counter!(
- "ingestion_consumer_transport_backpressure_waits_total",
- "worker" => self.worker_url.clone()
- )
- .increment(1);
- }
self.record_ledger_depth(ledger.len());
Ok(())
}
diff --git a/rust/ingestion-consumer/src/main.rs b/rust/ingestion-consumer/src/main.rs
index 48c7e3695e68..c55305f522ea 100644
--- a/rust/ingestion-consumer/src/main.rs
+++ b/rust/ingestion-consumer/src/main.rs
@@ -281,7 +281,6 @@ async fn async_main(config: Config) -> Result<()> {
let mut transport = GrpcTransport::new(
grpc_port,
- config.ingestion_worker_concurrent_batches,
Duration::from_millis(config.ingestion_worker_stream_ack_timeout_ms),
);
transport.set_max_body_bytes(config.transport_max_body_bytes);
diff --git a/rust/ingestion-consumer/tests/batcher_integration_test.rs b/rust/ingestion-consumer/tests/batcher_integration_test.rs
index 8a3ea12978ed..74fe73a6cd7d 100644
--- a/rust/ingestion-consumer/tests/batcher_integration_test.rs
+++ b/rust/ingestion-consumer/tests/batcher_integration_test.rs
@@ -45,7 +45,6 @@ async fn purging_a_just_submitted_batch_is_not_fatal() {
let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);
let transport = Arc::new(GrpcTransport::new(
GrpcPort::OffsetFromHttp(0),
- 1,
Duration::from_secs(30),
));
let (batcher, mut outputs) = one_message_batcher(
@@ -75,7 +74,6 @@ async fn dropping_an_idle_batcher_closes_its_outputs() {
let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);
let transport = Arc::new(GrpcTransport::new(
GrpcPort::OffsetFromHttp(0),
- 1,
Duration::from_secs(30),
));
let (batcher, mut outputs) = one_message_batcher(
diff --git a/rust/ingestion-consumer/tests/common/mod.rs b/rust/ingestion-consumer/tests/common/mod.rs
index 10de6fc0e015..6ff426b29665 100644
--- a/rust/ingestion-consumer/tests/common/mod.rs
+++ b/rust/ingestion-consumer/tests/common/mod.rs
@@ -27,8 +27,10 @@ pub const ONE_MESSAGE_PER_REQUEST: PackTargets = PackTargets {
latency_budget: Duration::ZERO,
};
-/// A batcher over `pool_source`'s workers, with the transport's un-acked cap
-/// as its per-worker request cap.
+/// Every suite keeps one request in flight per worker, so a scenario's second
+/// send to a worker waits for the first to settle.
+const ONE_REQUEST_PER_WORKER: usize = 1;
+
pub fn batcher(
pool_source: WorkerPoolSource,
transport: Arc<GrpcTransport>,
@@ -37,9 +39,8 @@ pub fn batcher(
pack_targets: PackTargets,
budget: Option<RequestBudget>,
) -> (Batcher, BatcherOutputs) {
- let assigner =
- WorkerAssigner::new(Router::new(pool_source.strategy()), transport.max_unacked())
- .expect("valid request cap");
+ let assigner = WorkerAssigner::new(Router::new(pool_source.strategy()), ONE_REQUEST_PER_WORKER)
+ .expect("valid request cap");
let retry = RetryPolicy::uniform(retry_delay).expect("valid retry delay");
let state_machine = BatcherStateMachine::new(
Packer::new(pack_targets),
diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rs
index d28543db0aa3..75a3c01641d3 100644
--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs
+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs
@@ -662,11 +662,11 @@ fn make_kafka_consumer_with_context(
}
/// The consumer's worker transport, addressed at each FakeWorker's gRPC port
-/// (`GrpcPort::OffsetFromHttp(1)` — HTTP port + 1). One un-acked sub-batch per
-/// worker, and a short ack timeout so a stalled scenario fences within a
-/// test's patience rather than production's 60s.
+/// (`GrpcPort::OffsetFromHttp(1)` — HTTP port + 1), with a short ack timeout
+/// so a stalled scenario fences within a test's patience rather than
+/// production's 60s.
fn test_transport() -> GrpcTransport {
- GrpcTransport::new(GrpcPort::OffsetFromHttp(1), 1, Duration::from_secs(10))
+ GrpcTransport::new(GrpcPort::OffsetFromHttp(1), Duration::from_secs(10))
}
/// Reap drained workers exactly as `main.rs` does in production: complete the
diff --git a/rust/ingestion-consumer/tests/grpc_transport_test.rs b/rust/ingestion-consumer/tests/grpc_transport_test.rs
index 2b226f8f2095..49a7f725179c 100644
--- a/rust/ingestion-consumer/tests/grpc_transport_test.rs
+++ b/rust/ingestion-consumer/tests/grpc_transport_test.rs
@@ -266,15 +266,11 @@ fn worker_url(addr: SocketAddr) -> String {
#[tokio::test]
async fn sub_batches_reach_the_worker_in_enqueue_order() {
let mock = start_mock(AckMode::Immediate, None).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
- // Enqueue three sub-batches back to back — more than the un-acked cap, so
- // ordering must survive the ledger pacing too.
+ // Enqueue three sub-batches back to back, so ordering must survive the
+ // queue draining onto the stream in one go.
let pending: Vec<_> = (0..3)
.map(|i| transport.begin_send(&url, &format!("batch-{i}"), vec![msg("d1", i)], false))
.collect();
@@ -297,11 +293,7 @@ async fn sub_batches_reach_the_worker_in_enqueue_order() {
async fn out_of_order_acks_resolve_the_right_sends() {
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1), msg("d1", 2)], false);
@@ -372,11 +364,7 @@ fn partial_ack(seq: u64, accepted: u32, unfinished: &[i64]) -> SubBatchAck {
async fn a_partial_ack_resolves_the_send_without_fencing_the_stream() {
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let budgeted = SendOptions {
replay: false,
@@ -439,11 +427,7 @@ async fn a_partial_ack_counts_a_message_the_sub_batch_carries_twice(
) {
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let budgeted = SendOptions {
replay: false,
@@ -497,11 +481,7 @@ async fn a_malformed_ack_fences_the_stream_as_a_fault(
) {
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let options = SendOptions {
replay: false,
@@ -541,11 +521,8 @@ async fn a_budget_rides_only_a_single_frame_send(
#[case] frame_budgets: Vec<u64>,
) {
let mock = start_mock(AckMode::Immediate, None).await;
- let mut transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 4,
- Duration::from_secs(30),
- );
+ let mut transport =
+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
transport.set_max_body_bytes(max_body_bytes);
let url = worker_url(mock.addr);
let options = SendOptions {
@@ -588,11 +565,7 @@ async fn a_failure_fences_a_later_sub_batch_the_worker_already_acked(
// ones. A failure must fence the acked-but-unresolved tail too.
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let budgeted = SendOptions {
replay: false,
@@ -636,11 +609,7 @@ async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {
// ahead of the requeued older messages. The worker stream must keep fencing until
// every fenced caller drops its guard.
let mock = start_mock(AckMode::NackSeq(1), None).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 1,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);
@@ -687,11 +656,8 @@ async fn an_oversized_sub_batch_rides_the_stream_as_ordered_chunks() {
// consecutive frames on the same stream instead of one frame the worker
// rejects, fences, and replays forever.
let mock = start_mock(AckMode::Immediate, None).await;
- let mut transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let mut transport =
+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
// Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this
// fits exactly one message per frame and three messages become three frames.
transport.set_max_body_bytes(200);
@@ -737,11 +703,8 @@ async fn a_failed_chunk_hands_back_the_whole_sub_batch() {
// the send fails with every message (acked ones included) for deferral,
// and carries a fence guard.
let mock = start_mock(AckMode::NackSeq(2), None).await;
- let mut transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let mut transport =
+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
// Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this
// fits exactly one message per frame and three messages become three frames.
transport.set_max_body_bytes(200);
@@ -771,11 +734,8 @@ async fn a_failed_chunk_hands_back_the_whole_sub_batch() {
async fn chunks_acked_out_of_order_then_fenced_hand_back_messages_in_send_order() {
let (ack_tx, ack_rx) = mpsc::unbounded_channel();
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
- let mut transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 3,
- Duration::from_secs(30),
- );
+ let mut transport =
+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
transport.set_max_body_bytes(200);
let url = worker_url(mock.addr);
@@ -806,15 +766,11 @@ async fn a_nack_fences_everything_outstanding_in_order() {
// nothing may be silently retried on the next stream — a later sub-batch
// surviving the fence would leapfrog the failed one and reorder its keys.
let mock = start_mock(AckMode::NackSeq(1), None).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 1,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
- // With max_unacked=1, the second and third wait in the queue behind the
- // first — the nack must fence all three.
+ // All three ride the stream at once, and the nack of the first must fence
+ // the other two in order.
let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);
let second = transport.begin_send(&url, "batch-2", vec![msg("d1", 2)], false);
let third = transport.begin_send(&url, "batch-3", vec![msg("d2", 3)], false);
@@ -840,20 +796,12 @@ async fn a_nack_fences_everything_outstanding_in_order() {
.await
.expect_err("queued send behind the nack must fail");
assert_eq!(third_err.messages[0].offset, 3);
-
- // Only the nacked sub-batch reached the worker; the fenced ones never
- // rode a stream.
- assert_eq!(mock.received.lock().await.len(), 1);
}
#[tokio::test]
async fn the_worker_stream_reconnects_with_a_new_stream_epoch_after_a_fence() {
let mock = start_mock(AckMode::NackSeq(1), None).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 2,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);
@@ -884,7 +832,7 @@ async fn a_dead_worker_fences_instead_of_hanging() {
// Connect failure (nothing listening): the send must resolve with its
// messages rather than waiting forever — the caller's deferral path owns
// the retry pacing.
- let transport = GrpcTransport::new(GrpcPort::Fixed(1), 1, Duration::from_secs(30));
+ let transport = GrpcTransport::new(GrpcPort::Fixed(1), Duration::from_secs(30));
let pending = transport.begin_send(
"http://127.0.0.1:9001",
"batch-1",
@@ -908,7 +856,6 @@ async fn a_worker_that_stops_acking_fences_after_the_watchdog_window() {
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
let transport = GrpcTransport::new(
GrpcPort::Fixed(mock.addr.port()),
- 2,
Duration::from_millis(200),
);
let url = worker_url(mock.addr);
@@ -933,7 +880,6 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {
let mock = start_mock(AckMode::AckExceptSeq(1), None).await;
let transport = Arc::new(GrpcTransport::new(
GrpcPort::Fixed(mock.addr.port()),
- 2,
Duration::from_millis(500),
));
let url = worker_url(mock.addr);
@@ -973,7 +919,34 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {
}
#[tokio::test]
-async fn removing_a_worker_fences_its_in_flight_send_with_messages() {
+async fn removing_a_worker_lets_its_in_flight_send_settle_by_ack() {
+ // Regression: a reaped worker that is slow rather than dead still finishes
+ // its request. Fencing the send on removal made a survivor repeat it while
+ // the reaped worker still ran it.
+ let (ack_tx, ack_rx) = mpsc::unbounded_channel();
+ let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
+ let url = worker_url(mock.addr);
+
+ let pending = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);
+ wait_for_received(&mock, 1).await;
+ transport.remove_worker(&url);
+ ack_tx
+ .send(ManualAck::Ok {
+ seq: 1,
+ accepted: 1,
+ })
+ .unwrap();
+
+ let outcome = tokio::time::timeout(Duration::from_secs(5), pending.wait())
+ .await
+ .expect("the removed worker stream still delivers the ack")
+ .expect("the ack settles the send");
+ assert!(matches!(outcome, SendOutcome::Accepted(1)), "{outcome:?}");
+}
+
+#[tokio::test]
+async fn removing_a_worker_resolves_its_unacked_send_by_the_watchdog() {
// Regression: reaping a worker that still has in-flight work must resolve
// its un-acked sends with the messages intact, so the deferral path can
// replay them. Aborting the worker stream task instead dropped the sends unresolved,
@@ -982,7 +955,6 @@ async fn removing_a_worker_fences_its_in_flight_send_with_messages() {
let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;
let transport = GrpcTransport::new(
GrpcPort::Fixed(mock.addr.port()),
- 2,
Duration::from_millis(200),
);
let url = worker_url(mock.addr);
@@ -1005,8 +977,8 @@ async fn removing_a_worker_fences_its_in_flight_send_with_messages() {
let err = tokio::time::timeout(Duration::from_secs(5), pending.wait())
.await
- .expect("reaped worker stream must fence, not hang")
- .expect_err("reaped worker fences the in-flight send");
+ .expect("reaped worker stream must fence at the ack timeout, not hang")
+ .expect_err("the ack watchdog fences the unacked send");
assert_eq!(err.messages.len(), 1, "messages come back for the requeue");
assert_eq!(err.messages[0].offset, 1);
}
@@ -1017,11 +989,7 @@ async fn a_busy_status_fences_as_retriable_with_messages() {
// sends must carry their messages (to replay) and classify as retriable, so
// the consumer re-routes without counting the worker as unhealthy.
let mock = start_mock(AckMode::BusySeq(1), None).await;
- let transport = GrpcTransport::new(
- GrpcPort::Fixed(mock.addr.port()),
- 1,
- Duration::from_secs(30),
- );
+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));
let url = worker_url(mock.addr);
let pending = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);
@@ -1150,7 +1118,6 @@ async fn the_watchdog_bounds_overlapping_busy_retries() {
let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);
let transport = Arc::new(GrpcTransport::new(
GrpcPort::OffsetFromHttp(0),
- 1,
Duration::from_secs(30),
));
let (batcher, mut outputs) = one_message_batcher(
@@ -1234,7 +1201,6 @@ async fn a_busy_send_is_retried_until_it_completes() {
let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);
let transport = Arc::new(GrpcTransport::new(
GrpcPort::OffsetFromHttp(0),
- 1,
Duration::from_secs(30),
));
let (batcher, mut outputs) = one_message_batcher(
@@ -1280,7 +1246,6 @@ async fn the_watchdog_allows_in_flight_success_after_the_deadline() {
let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);
let transport = Arc::new(GrpcTransport::new(
GrpcPort::OffsetFromHttp(0),
- 1,
Duration::from_secs(30),
));
let (batcher, mut outputs) = one_message_batcher(