:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ingestion-consumer): remove the transport per-worker cap (#121245)
José Sequeira25027f9ed1c3
diff --git a/rust/ingestion-consumer/src/config.rs b/rust/ingestion-consumer/src/config.rsindex 5639e13547bb..de6d02bb5092 100644--- a/rust/ingestion-consumer/src/config.rs+++ b/rust/ingestion-consumer/src/config.rs@@ -326,10 +326,11 @@ pub struct Config {#[envconfig(default = "http://localhost:9001")]pub worker_addresses: String,- /// Cap on un-acked sub-batches per worker stream. Ideally aligned with the- /// worker's `BatchingPipeline.concurrentBatches`- /// (`INGESTION_WORKER_CONCURRENT_BATCHES` on the Node.js side) so the- /// stream backpressures before the worker stalls its reads.+ /// The most requests the batcher keeps in flight on one worker, and the+ /// governor's permits per worker at its ceiling. Aligned with the worker's+ /// `BatchingPipeline.concurrentBatches` (`INGESTION_WORKER_CONCURRENT_BATCHES`+ /// on the Node.js side), because the worker queues requests past it+ /// without saying so.#[envconfig(from = "INGESTION_WORKER_CONCURRENT_BATCHES", default = "1")]pub ingestion_worker_concurrent_batches: usize,@@ -512,9 +513,8 @@ impl Config {format!("{}:{}", self.bind_host, self.bind_port)}- /// The batcher state machine. It reuses the stream's un-acked cap as its- /// per-worker request cap, the parked-retry interval for every retry, and- /// the deferred-flush timeout as its stall timeout.+ /// The batcher state machine. It takes the parked-retry interval for every+ /// retry and the deferred-flush timeout as its stall timeout.pub fn batcher_state_machine(&self) -> Result<BatcherStateMachine, String> {let mut packer = Packer::new(PackTargets {events: NonZeroUsize::new(self.pack_target_events),diff --git a/rust/ingestion-consumer/src/grpc_transport.rs b/rust/ingestion-consumer/src/grpc_transport.rsindex fc96fb878eb5..b09f690bfc3b 100644--- a/rust/ingestion-consumer/src/grpc_transport.rs+++ b/rust/ingestion-consumer/src/grpc_transport.rs@@ -31,11 +31,10 @@//! Nothing for a fenced key can leapfrog the failure, because everything for//! it was either in the ledger, the queue, or fenced on arrival.//!-//! Backpressure: the queue is unbounded (so enqueue stays synchronous and-//! ordered) but the worker stream keeps at most `max_unacked` sub-batches un-acked on-//! the stream; queue depth is bounded in practice by the consumer's-//! `max_in_flight_batches`, whose batches cannot complete until their sends-//! resolve.+//! The queue is unbounded, so enqueue stays synchronous and ordered. The+//! batcher bounds its depth: it keeps at most `INGESTION_WORKER_CONCURRENT_BATCHES`+//! requests in flight per worker and sends nothing more until one settles,+//! so every queued sub-batch goes on the wire at once.use std::collections::{HashMap, VecDeque};use std::sync::Arc;@@ -194,9 +193,6 @@ pub struct GrpcTransport {consumer_id: String,/// How each worker's stream address is derived from its HTTP URL.grpc_port: GrpcPort,- /// Max un-acked sub-batches per worker stream (aligned with the worker's- /// `concurrentBatches`).- max_unacked: usize,/// Fence the worker stream when un-acked work sees no ack for this long.ack_timeout: Duration,/// Bumped on Kafka partition assignment; stamped on every sub-batch so@@ -211,13 +207,11 @@ pub struct GrpcTransport {}impl GrpcTransport {- pub fn new(grpc_port: GrpcPort, max_unacked: usize, ack_timeout: Duration) -> Self {- assert!(max_unacked > 0, "max_unacked must be > 0");+ pub fn new(grpc_port: GrpcPort, ack_timeout: Duration) -> Self {Self {worker_streams: DashMap::new(),consumer_id: make_consumer_id(),grpc_port,- max_unacked,ack_timeout,max_body_bytes: DEFAULT_MAX_BODY_BYTES,// A fresh epoch for standalone construction; production wiring@@ -237,11 +231,6 @@ impl GrpcTransport {self.assignment_epoch = epoch;}- /// The most sub-batches one worker stream keeps un-acked.- pub fn max_unacked(&self) -> usize {- self.max_unacked- }-/// The assignment epoch this transport stamps on sub-batches.pub fn assignment_epoch(&self) -> AssignmentEpoch {self.assignment_epoch.clone()@@ -345,7 +334,6 @@ impl GrpcTransport {worker_url: Arc::from(worker_url),grpc_url: grpc_url(worker_url, self.grpc_port),consumer_id: self.consumer_id.clone(),- max_unacked: self.max_unacked,ack_timeout: self.ack_timeout,assignment_epoch: self.assignment_epoch.clone(),};@@ -354,10 +342,12 @@ impl GrpcTransport {}/// Tear down a departed worker's stream. Dropping it closes its queue- /// sender, so the runner fences whatever is still in flight **in order**,- /// with the messages intact, and exits on its own. Aborting the task would- /// instead drop the un-acked sends unresolved, which the caller can only- /// recover by crashing and replaying — so let the graceful path run.+ /// sender: the runner takes no more work, lets the sub-batches already on+ /// the wire settle by ack or by the ack watchdog, and exits on its own.+ /// A reaped worker that is slow rather than dead so finishes its request+ /// instead of a survivor repeating it while it still runs. Aborting the+ /// task would instead drop the un-acked sends unresolved, which the caller+ /// can only recover by crashing and replaying.pub fn remove_worker(&self, worker_url: &str) {self.worker_streams.remove(worker_url);}@@ -410,7 +400,6 @@ struct WorkerStreamRunner {worker_url: Arc<str>,grpc_url: String,consumer_id: String,- max_unacked: usize,ack_timeout: Duration,assignment_epoch: AssignmentEpoch,}@@ -582,8 +571,9 @@ impl WorkerStreamRunner {}}- /// Send the held first item, then pull more only while the ledger has- /// room — the worker-aligned concurrency cap. `Err` ends the stream.+ /// The batcher caps the requests in flight per worker, so nothing is held+ /// back here. A closed queue stops the filling; the ledger still drains by+ /// ack. `Err` ends the stream.fn fill_ledger(&self,out_tx: &mpsc::UnboundedSender<IngestStreamRequest>,@@ -592,19 +582,17 @@ impl WorkerStreamRunner {ledger: &mut VecDeque<LedgerEntry>,next_seq: &mut u64,) -> Result<(), StreamEnd> {- while ledger.len() < self.max_unacked {+ loop {let item = match pending_first.take() {Some(item) => Some(item),None => match queue.try_recv() {Ok(item) => Some(item),Err(mpsc::error::TryRecvError::Empty) => None,Err(mpsc::error::TryRecvError::Disconnected) => {- return Err(if ledger.is_empty() {- StreamEnd::QueueClosed- } else {- // Resolve the tail before exiting.- self.fence(None, ledger, queue, "queue closed")- });+ if ledger.is_empty() {+ return Err(StreamEnd::QueueClosed);+ }+ None}},};@@ -654,8 +642,9 @@ impl WorkerStreamRunner {.record(entry.sent_at.elapsed().as_secs_f64());}- /// Wait for an ack, or for new work when the ledger has room. `Ok(None)`- /// means new work was held in `pending_first`; `Err` ends the stream.+ /// Wait for an ack, or for new work while the queue is open. `Ok(None)`+ /// means new work was held in `pending_first`, or the queue closed under+ /// the wait and the ledger still drains; `Err` ends the stream.////// The ack-progress watchdog bounds how long the oldest un-acked/// sub-batch may sit unacked. Each entry carries its own deadline, armed@@ -692,12 +681,9 @@ impl WorkerStreamRunner {);Err(self.fence(pending_first.take(), ledger, queue, "ack progress timeout"))}- item = queue.recv(), if ledger.len() < self.max_unacked => match item {- Some(item) => {- *pending_first = Some(item);- Ok(None)- }- None => Err(self.fence(None, ledger, queue, "queue closed")),+ item = queue.recv(), if !queue.is_closed() => {+ *pending_first = item;+ Ok(None)}}}@@ -754,7 +740,6 @@ impl WorkerStreamRunner {);return Err(self.fence_with(true, None, ledger, queue, "worker busy"));}- let was_full = ledger.len() >= self.max_unacked;let Some(index) = ledger.iter().position(|e| e.seq == response.seq && e.acked.is_none())@@ -801,13 +786,6 @@ impl WorkerStreamRunner {messages,}));}- if was_full && ledger.len() < self.max_unacked && !queue.is_empty() {- counter!(- "ingestion_consumer_transport_backpressure_waits_total",- "worker" => self.worker_url.clone()- )- .increment(1);- }self.record_ledger_depth(ledger.len());Ok(())}diff --git a/rust/ingestion-consumer/src/main.rs b/rust/ingestion-consumer/src/main.rsindex 48c7e3695e68..c55305f522ea 100644--- a/rust/ingestion-consumer/src/main.rs+++ b/rust/ingestion-consumer/src/main.rs@@ -281,7 +281,6 @@ async fn async_main(config: Config) -> Result<()> {let mut transport = GrpcTransport::new(grpc_port,- config.ingestion_worker_concurrent_batches,Duration::from_millis(config.ingestion_worker_stream_ack_timeout_ms),);transport.set_max_body_bytes(config.transport_max_body_bytes);diff --git a/rust/ingestion-consumer/tests/batcher_integration_test.rs b/rust/ingestion-consumer/tests/batcher_integration_test.rsindex 8a3ea12978ed..74fe73a6cd7d 100644--- a/rust/ingestion-consumer/tests/batcher_integration_test.rs+++ b/rust/ingestion-consumer/tests/batcher_integration_test.rs@@ -45,7 +45,6 @@ async fn purging_a_just_submitted_batch_is_not_fatal() {let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);let transport = Arc::new(GrpcTransport::new(GrpcPort::OffsetFromHttp(0),- 1,Duration::from_secs(30),));let (batcher, mut outputs) = one_message_batcher(@@ -75,7 +74,6 @@ async fn dropping_an_idle_batcher_closes_its_outputs() {let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);let transport = Arc::new(GrpcTransport::new(GrpcPort::OffsetFromHttp(0),- 1,Duration::from_secs(30),));let (batcher, mut outputs) = one_message_batcher(diff --git a/rust/ingestion-consumer/tests/common/mod.rs b/rust/ingestion-consumer/tests/common/mod.rsindex 10de6fc0e015..6ff426b29665 100644--- a/rust/ingestion-consumer/tests/common/mod.rs+++ b/rust/ingestion-consumer/tests/common/mod.rs@@ -27,8 +27,10 @@ pub const ONE_MESSAGE_PER_REQUEST: PackTargets = PackTargets {latency_budget: Duration::ZERO,};-/// A batcher over `pool_source`'s workers, with the transport's un-acked cap-/// as its per-worker request cap.+/// Every suite keeps one request in flight per worker, so a scenario's second+/// send to a worker waits for the first to settle.+const ONE_REQUEST_PER_WORKER: usize = 1;+pub fn batcher(pool_source: WorkerPoolSource,transport: Arc<GrpcTransport>,@@ -37,9 +39,8 @@ pub fn batcher(pack_targets: PackTargets,budget: Option<RequestBudget>,) -> (Batcher, BatcherOutputs) {- let assigner =- WorkerAssigner::new(Router::new(pool_source.strategy()), transport.max_unacked())- .expect("valid request cap");+ let assigner = WorkerAssigner::new(Router::new(pool_source.strategy()), ONE_REQUEST_PER_WORKER)+ .expect("valid request cap");let retry = RetryPolicy::uniform(retry_delay).expect("valid retry delay");let state_machine = BatcherStateMachine::new(Packer::new(pack_targets),diff --git a/rust/ingestion-consumer/tests/e2e_integration_test.rs b/rust/ingestion-consumer/tests/e2e_integration_test.rsindex d28543db0aa3..75a3c01641d3 100644--- a/rust/ingestion-consumer/tests/e2e_integration_test.rs+++ b/rust/ingestion-consumer/tests/e2e_integration_test.rs@@ -662,11 +662,11 @@ fn make_kafka_consumer_with_context(}/// The consumer's worker transport, addressed at each FakeWorker's gRPC port-/// (`GrpcPort::OffsetFromHttp(1)` — HTTP port + 1). One un-acked sub-batch per-/// worker, and a short ack timeout so a stalled scenario fences within a-/// test's patience rather than production's 60s.+/// (`GrpcPort::OffsetFromHttp(1)` — HTTP port + 1), with a short ack timeout+/// so a stalled scenario fences within a test's patience rather than+/// production's 60s.fn test_transport() -> GrpcTransport {- GrpcTransport::new(GrpcPort::OffsetFromHttp(1), 1, Duration::from_secs(10))+ GrpcTransport::new(GrpcPort::OffsetFromHttp(1), Duration::from_secs(10))}/// Reap drained workers exactly as `main.rs` does in production: complete thediff --git a/rust/ingestion-consumer/tests/grpc_transport_test.rs b/rust/ingestion-consumer/tests/grpc_transport_test.rsindex 2b226f8f2095..49a7f725179c 100644--- a/rust/ingestion-consumer/tests/grpc_transport_test.rs+++ b/rust/ingestion-consumer/tests/grpc_transport_test.rs@@ -266,15 +266,11 @@ fn worker_url(addr: SocketAddr) -> String {#[tokio::test]async fn sub_batches_reach_the_worker_in_enqueue_order() {let mock = start_mock(AckMode::Immediate, None).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);- // Enqueue three sub-batches back to back — more than the un-acked cap, so- // ordering must survive the ledger pacing too.+ // Enqueue three sub-batches back to back, so ordering must survive the+ // queue draining onto the stream in one go.let pending: Vec<_> = (0..3).map(|i| transport.begin_send(&url, &format!("batch-{i}"), vec![msg("d1", i)], false)).collect();@@ -297,11 +293,7 @@ async fn sub_batches_reach_the_worker_in_enqueue_order() {async fn out_of_order_acks_resolve_the_right_sends() {let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1), msg("d1", 2)], false);@@ -372,11 +364,7 @@ fn partial_ack(seq: u64, accepted: u32, unfinished: &[i64]) -> SubBatchAck {async fn a_partial_ack_resolves_the_send_without_fencing_the_stream() {let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let budgeted = SendOptions {replay: false,@@ -439,11 +427,7 @@ async fn a_partial_ack_counts_a_message_the_sub_batch_carries_twice() {let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let budgeted = SendOptions {replay: false,@@ -497,11 +481,7 @@ async fn a_malformed_ack_fences_the_stream_as_a_fault() {let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let options = SendOptions {replay: false,@@ -541,11 +521,8 @@ async fn a_budget_rides_only_a_single_frame_send(#[case] frame_budgets: Vec<u64>,) {let mock = start_mock(AckMode::Immediate, None).await;- let mut transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 4,- Duration::from_secs(30),- );+ let mut transport =+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));transport.set_max_body_bytes(max_body_bytes);let url = worker_url(mock.addr);let options = SendOptions {@@ -588,11 +565,7 @@ async fn a_failure_fences_a_later_sub_batch_the_worker_already_acked(// ones. A failure must fence the acked-but-unresolved tail too.let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let budgeted = SendOptions {replay: false,@@ -636,11 +609,7 @@ async fn sends_enqueued_during_a_fence_are_fenced_until_the_callers_requeue() {// ahead of the requeued older messages. The worker stream must keep fencing until// every fenced caller drops its guard.let mock = start_mock(AckMode::NackSeq(1), None).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 1,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);@@ -687,11 +656,8 @@ async fn an_oversized_sub_batch_rides_the_stream_as_ordered_chunks() {// consecutive frames on the same stream instead of one frame the worker// rejects, fences, and replays forever.let mock = start_mock(AckMode::Immediate, None).await;- let mut transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let mut transport =+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));// Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this// fits exactly one message per frame and three messages become three frames.transport.set_max_body_bytes(200);@@ -737,11 +703,8 @@ async fn a_failed_chunk_hands_back_the_whole_sub_batch() {// the send fails with every message (acked ones included) for deferral,// and carries a fence guard.let mock = start_mock(AckMode::NackSeq(2), None).await;- let mut transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let mut transport =+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));// Frame size cap in bytes. One `msg()` estimates to ~143 bytes, so this// fits exactly one message per frame and three messages become three frames.transport.set_max_body_bytes(200);@@ -771,11 +734,8 @@ async fn a_failed_chunk_hands_back_the_whole_sub_batch() {async fn chunks_acked_out_of_order_then_fenced_hand_back_messages_in_send_order() {let (ack_tx, ack_rx) = mpsc::unbounded_channel();let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;- let mut transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 3,- Duration::from_secs(30),- );+ let mut transport =+ GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));transport.set_max_body_bytes(200);let url = worker_url(mock.addr);@@ -806,15 +766,11 @@ async fn a_nack_fences_everything_outstanding_in_order() {// nothing may be silently retried on the next stream — a later sub-batch// surviving the fence would leapfrog the failed one and reorder its keys.let mock = start_mock(AckMode::NackSeq(1), None).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 1,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);- // With max_unacked=1, the second and third wait in the queue behind the- // first — the nack must fence all three.+ // All three ride the stream at once, and the nack of the first must fence+ // the other two in order.let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);let second = transport.begin_send(&url, "batch-2", vec![msg("d1", 2)], false);let third = transport.begin_send(&url, "batch-3", vec![msg("d2", 3)], false);@@ -840,20 +796,12 @@ async fn a_nack_fences_everything_outstanding_in_order() {.await.expect_err("queued send behind the nack must fail");assert_eq!(third_err.messages[0].offset, 3);-- // Only the nacked sub-batch reached the worker; the fenced ones never- // rode a stream.- assert_eq!(mock.received.lock().await.len(), 1);}#[tokio::test]async fn the_worker_stream_reconnects_with_a_new_stream_epoch_after_a_fence() {let mock = start_mock(AckMode::NackSeq(1), None).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 2,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let first = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);@@ -884,7 +832,7 @@ async fn a_dead_worker_fences_instead_of_hanging() {// Connect failure (nothing listening): the send must resolve with its// messages rather than waiting forever — the caller's deferral path owns// the retry pacing.- let transport = GrpcTransport::new(GrpcPort::Fixed(1), 1, Duration::from_secs(30));+ let transport = GrpcTransport::new(GrpcPort::Fixed(1), Duration::from_secs(30));let pending = transport.begin_send("http://127.0.0.1:9001","batch-1",@@ -908,7 +856,6 @@ async fn a_worker_that_stops_acking_fences_after_the_watchdog_window() {let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()),- 2,Duration::from_millis(200),);let url = worker_url(mock.addr);@@ -933,7 +880,6 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {let mock = start_mock(AckMode::AckExceptSeq(1), None).await;let transport = Arc::new(GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()),- 2,Duration::from_millis(500),));let url = worker_url(mock.addr);@@ -973,7 +919,34 @@ async fn a_stuck_oldest_sub_batch_fences_even_while_siblings_keep_acking() {}#[tokio::test]-async fn removing_a_worker_fences_its_in_flight_send_with_messages() {+async fn removing_a_worker_lets_its_in_flight_send_settle_by_ack() {+ // Regression: a reaped worker that is slow rather than dead still finishes+ // its request. Fencing the send on removal made a survivor repeat it while+ // the reaped worker still ran it.+ let (ack_tx, ack_rx) = mpsc::unbounded_channel();+ let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));+ let url = worker_url(mock.addr);++ let pending = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);+ wait_for_received(&mock, 1).await;+ transport.remove_worker(&url);+ ack_tx+ .send(ManualAck::Ok {+ seq: 1,+ accepted: 1,+ })+ .unwrap();++ let outcome = tokio::time::timeout(Duration::from_secs(5), pending.wait())+ .await+ .expect("the removed worker stream still delivers the ack")+ .expect("the ack settles the send");+ assert!(matches!(outcome, SendOutcome::Accepted(1)), "{outcome:?}");+}++#[tokio::test]+async fn removing_a_worker_resolves_its_unacked_send_by_the_watchdog() {// Regression: reaping a worker that still has in-flight work must resolve// its un-acked sends with the messages intact, so the deferral path can// replay them. Aborting the worker stream task instead dropped the sends unresolved,@@ -982,7 +955,6 @@ async fn removing_a_worker_fences_its_in_flight_send_with_messages() {let mock = start_mock(AckMode::Manual, Some(ack_rx)).await;let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()),- 2,Duration::from_millis(200),);let url = worker_url(mock.addr);@@ -1005,8 +977,8 @@ async fn removing_a_worker_fences_its_in_flight_send_with_messages() {let err = tokio::time::timeout(Duration::from_secs(5), pending.wait()).await- .expect("reaped worker stream must fence, not hang")- .expect_err("reaped worker fences the in-flight send");+ .expect("reaped worker stream must fence at the ack timeout, not hang")+ .expect_err("the ack watchdog fences the unacked send");assert_eq!(err.messages.len(), 1, "messages come back for the requeue");assert_eq!(err.messages[0].offset, 1);}@@ -1017,11 +989,7 @@ async fn a_busy_status_fences_as_retriable_with_messages() {// sends must carry their messages (to replay) and classify as retriable, so// the consumer re-routes without counting the worker as unhealthy.let mock = start_mock(AckMode::BusySeq(1), None).await;- let transport = GrpcTransport::new(- GrpcPort::Fixed(mock.addr.port()),- 1,- Duration::from_secs(30),- );+ let transport = GrpcTransport::new(GrpcPort::Fixed(mock.addr.port()), Duration::from_secs(30));let url = worker_url(mock.addr);let pending = transport.begin_send(&url, "batch-1", vec![msg("d1", 1)], false);@@ -1150,7 +1118,6 @@ async fn the_watchdog_bounds_overlapping_busy_retries() {let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);let transport = Arc::new(GrpcTransport::new(GrpcPort::OffsetFromHttp(0),- 1,Duration::from_secs(30),));let (batcher, mut outputs) = one_message_batcher(@@ -1234,7 +1201,6 @@ async fn a_busy_send_is_retried_until_it_completes() {let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);let transport = Arc::new(GrpcTransport::new(GrpcPort::OffsetFromHttp(0),- 1,Duration::from_secs(30),));let (batcher, mut outputs) = one_message_batcher(@@ -1280,7 +1246,6 @@ async fn the_watchdog_allows_in_flight_success_after_the_deadline() {let pool_source = WorkerPoolSource::new(registry, RoutingStrategy::P2c);let transport = Arc::new(GrpcTransport::new(GrpcPort::OffsetFromHttp(0),- 1,Duration::from_secs(30),));let (batcher, mut outputs) = one_message_batcher(