Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(cohorts): recover test readers from kafka transport errors (#127882)

Paul D'Ambra3eb2287bc419
diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml
index 606801d5adf3..10cd7fde3294 100644
--- a/.github/workflows/ci-rust.yml
+++ b/.github/workflows/ci-rust.yml
@@ -290,7 +290,9 @@ jobs:
working-directory: .
run: |
# Use docker to clean up files created by containers (from repo root, not rust/)
- [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
+ if [ -d "data" ]; then
+ docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true
+ fi
continue-on-error: true
- name: Log in to Docker Hub
@@ -317,13 +319,6 @@ jobs:
echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts
- - name: Dump Kafka logs on failure
- if: failure()
- run: |
- docker ps -a || true
- docker logs --tail=500 rust-kafka-1 || true
- docker inspect rust-kafka-1 || true
-
# please keep the tag version here in sync with rust-version in rust/*/Cargo.toml
- name: Install rust
uses: ./.github/actions/setup-rust
@@ -496,6 +491,15 @@ jobs:
done
exit $overall_rc
+ - name: Dump Kafka logs on failure
+ if: ${{ !cancelled() && (failure() || steps.run-tests.outcome == 'failure') }}
+ env:
+ COMPOSE_FILE: ../docker-compose.dev.yml:../docker-compose.profiles.yml
+ run: |
+ docker ps -a || true
+ docker compose logs --no-color --tail=500 kafka || true
+ docker compose ps -a -q kafka | xargs -r docker inspect || true
+
# Doctests fail the job directly: they emit no JUnit, so the quarantine gate
# below can neither see nor mask them.
- name: Run doctests
diff --git a/rust/cohort-stream-processor/tests/cascade_consumer.rs b/rust/cohort-stream-processor/tests/cascade_consumer.rs
index d6ecb49c0222..6d7b4e398d79 100644
--- a/rust/cohort-stream-processor/tests/cascade_consumer.rs
+++ b/rust/cohort-stream-processor/tests/cascade_consumer.rs
@@ -13,6 +13,8 @@
//!
//! Each test deletes its topics on the way out via `with_topics_cleanup`.
+mod support;
+
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::panic::AssertUnwindSafe;
@@ -52,15 +54,16 @@ use lifecycle::{ComponentOptions, Handle, Manager};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
use rdkafka::consumer::{Consumer, StreamConsumer};
-use rdkafka::message::Message;
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::util::Timeout;
-use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+use rdkafka::ClientConfig;
use serde_json::json;
use tempfile::TempDir;
use tokio::task::JoinHandle;
use uuid::Uuid;
+use support::{bootstrap_servers, consume_all};
+
const TEAM: i32 = 7;
const A_HASH: &str = "aaaaaaaaaaaaaaaa";
const NUM_PARTITIONS: i32 = COHORT_PARTITION_COUNT as i32;
@@ -68,10 +71,6 @@ const TS: &str = "2026-06-10 12:34:56.789000";
const COMMIT_INTERVAL: Duration = Duration::from_millis(250);
const RECV_TIMEOUT: Duration = Duration::from_millis(200);
-fn bootstrap_servers() -> String {
- std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
-}
-
/// Cohort 2 (A): single `performed_event` on `$a`. Cohort 1 (B): pure cohort-ref to A.
/// With the cascade gate on, B is `Stage2ComposableRef` — only a cascade from A can flip it,
/// never its own event.
@@ -240,38 +239,6 @@ fn topic_message_count(topic: &str) -> i64 {
.sum()
}
-/// Read up to `expected` messages (all partitions from the beginning). Assigned explicitly — no group
-/// join, no commit.
-async fn consume_all(topic: &str, expected: usize, deadline: Duration) -> Vec<(i32, Vec<u8>)> {
- let consumer: StreamConsumer = ClientConfig::new()
- .set("bootstrap.servers", bootstrap_servers())
- .set("group.id", format!("scan-verifier-{}", Uuid::new_v4()))
- .set("enable.auto.commit", "false")
- .create()
- .expect("create scan verifier");
- let mut tpl = TopicPartitionList::new();
- for partition in 0..NUM_PARTITIONS {
- tpl.add_partition_offset(topic, partition, Offset::Beginning)
- .expect("add partition to scan TPL");
- }
- consumer.assign(&tpl).expect("assign scan verifier");
-
- let mut messages = Vec::new();
- let start = Instant::now();
- while messages.len() < expected && start.elapsed() < deadline {
- match tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await {
- Ok(Ok(message)) => {
- if let Some(payload) = message.payload() {
- messages.push((message.partition(), payload.to_vec()));
- }
- }
- Ok(Err(err)) => panic!("scan verifier recv error: {err}"),
- Err(_) => {}
- }
- }
- messages
-}
-
async fn wait_for(what: &str, deadline: Duration, mut condition: impl FnMut() -> bool) {
let start = Instant::now();
while !condition() {
diff --git a/rust/cohort-stream-processor/tests/events_consumer.rs b/rust/cohort-stream-processor/tests/events_consumer.rs
index 07da8f75355f..f9f63ed9a026 100644
--- a/rust/cohort-stream-processor/tests/events_consumer.rs
+++ b/rust/cohort-stream-processor/tests/events_consumer.rs
@@ -28,6 +28,8 @@
// Tests seed and assert through `CohortStore` directly — the sanctioned direct-store test surface.
#![allow(clippy::disallowed_methods)]
+mod support;
+
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::num::NonZeroU16;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -87,6 +89,8 @@ use tempfile::TempDir;
use tokio::sync::mpsc;
use uuid::Uuid;
+use support::bootstrap_servers;
+
const TEAM: i32 = 7;
const HASH: [u8; 16] = *b"0123456789abcdef";
const NUM_PARTITIONS: i32 = 4;
@@ -94,10 +98,6 @@ const PERSONS: u128 = 3;
const EVENTS_PER_PERSON: usize = 3;
const BASE_TS: &str = "2026-05-26 12:34:56.789000";
-fn bootstrap_servers() -> String {
- std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
-}
-
/// A team with a single `performed_event` behavioral leaf on `$pageview` — every `$pageview` enters.
fn behavioral_catalog() -> CatalogHandle {
let leaf = json!({
@@ -393,7 +393,12 @@ async fn drain_shadow_changes(topic: &str, expected: usize) -> Vec<CohortMembers
let mut changes = Vec::new();
let start = Instant::now();
while changes.len() < expected && start.elapsed() < Duration::from_secs(30) {
- match tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await {
+ match tokio::time::timeout(
+ Duration::from_secs(2),
+ support::recv(&mut consumer.stream()),
+ )
+ .await
+ {
Ok(Ok(message)) => {
if let Some(payload) = message.payload() {
changes.push(
diff --git a/rust/cohort-stream-processor/tests/kafka_reader.rs b/rust/cohort-stream-processor/tests/kafka_reader.rs
new file mode 100644
index 000000000000..7e84509b3c5b
--- /dev/null
+++ b/rust/cohort-stream-processor/tests/kafka_reader.rs
@@ -0,0 +1,55 @@
+mod support;
+
+use std::time::Duration;
+
+use futures::stream;
+use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+
+#[tokio::test(start_paused = true)]
+async fn transport_errors_do_not_discard_messages() {
+ use RDKafkaErrorCode::{AllBrokersDown, BrokerTransportFailure};
+
+ for codes in [
+ &[][..],
+ &[BrokerTransportFailure],
+ &[BrokerTransportFailure; 3],
+ &[AllBrokersDown],
+ &[BrokerTransportFailure, AllBrokersDown],
+ ] {
+ let errors = codes
+ .iter()
+ .map(|&code| Err(KafkaError::MessageConsumption(code)));
+ let mut messages = stream::iter(errors.chain([Ok(1), Ok(2)]));
+
+ assert_eq!(support::recv(&mut messages).await.unwrap(), 1);
+ assert_eq!(support::recv(&mut messages).await.unwrap(), 2);
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn other_errors_fail_without_waiting() {
+ for error in [
+ KafkaError::MessageConsumption(RDKafkaErrorCode::TopicAuthorizationFailed),
+ KafkaError::MessageConsumption(RDKafkaErrorCode::UnknownTopicOrPartition),
+ KafkaError::MessageConsumptionFatal(RDKafkaErrorCode::BrokerTransportFailure),
+ ] {
+ let mut messages = stream::iter([Err::<(), _>(error.clone()), Ok(())]);
+ let start = tokio::time::Instant::now();
+
+ assert_eq!(support::recv(&mut messages).await, Err(error));
+ assert_eq!(start.elapsed(), Duration::ZERO);
+ }
+}
+
+#[tokio::test(start_paused = true)]
+async fn repeated_transport_errors_obey_the_callers_deadline() {
+ let error = KafkaError::MessageConsumption(RDKafkaErrorCode::BrokerTransportFailure);
+ let mut messages = stream::repeat(Err::<(), _>(error));
+ let deadline = Duration::from_secs(1);
+ let start = tokio::time::Instant::now();
+
+ assert!(tokio::time::timeout(deadline, support::recv(&mut messages))
+ .await
+ .is_err());
+ assert_eq!(start.elapsed(), deadline);
+}
diff --git a/rust/cohort-stream-processor/tests/merge_consumer.rs b/rust/cohort-stream-processor/tests/merge_consumer.rs
index 86ee2faf92e2..3521be3a384a 100644
--- a/rust/cohort-stream-processor/tests/merge_consumer.rs
+++ b/rust/cohort-stream-processor/tests/merge_consumer.rs
@@ -23,6 +23,8 @@
// Tests seed and assert through `CohortStore` directly — the sanctioned direct-store test surface.
#![allow(clippy::disallowed_methods)]
+mod support;
+
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::panic::AssertUnwindSafe;
@@ -76,6 +78,8 @@ use tempfile::TempDir;
use tokio::task::JoinHandle;
use uuid::Uuid;
+use support::{bootstrap_servers, consume_all};
+
const TEAM: i32 = 7;
const HASH: [u8; 16] = *b"0123456789abcdef";
const NUM_PARTITIONS: i32 = COHORT_PARTITION_COUNT as i32;
@@ -85,10 +89,6 @@ const MERGED_AT: i64 = 1_770_000_000_000;
const COMMIT_INTERVAL: Duration = Duration::from_millis(250);
const RECV_TIMEOUT: Duration = Duration::from_millis(200);
-fn bootstrap_servers() -> String {
- std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
-}
-
/// Cohort 1: `performed_event` single. Cohort 2: `performed_event_multiple gte 2` daily.
/// Both share one `$pageview` conditionHash; daily bucket counts make a double-apply observable.
fn merge_catalog() -> CatalogHandle {
@@ -306,38 +306,6 @@ fn topic_message_count(topic: &str) -> i64 {
.sum()
}
-/// Read up to `expected` messages (all partitions from the beginning). Assigned explicitly — no group
-/// join, no commit — so committed offsets the test asserts on are never perturbed.
-async fn consume_all(topic: &str, expected: usize, deadline: Duration) -> Vec<(i32, Vec<u8>)> {
- let consumer: StreamConsumer = ClientConfig::new()
- .set("bootstrap.servers", bootstrap_servers())
- .set("group.id", format!("scan-verifier-{}", Uuid::new_v4()))
- .set("enable.auto.commit", "false")
- .create()
- .expect("create scan verifier");
- let mut tpl = TopicPartitionList::new();
- for partition in 0..NUM_PARTITIONS {
- tpl.add_partition_offset(topic, partition, Offset::Beginning)
- .expect("add partition to scan TPL");
- }
- consumer.assign(&tpl).expect("assign scan verifier");
-
- let mut messages = Vec::new();
- let start = Instant::now();
- while messages.len() < expected && start.elapsed() < deadline {
- match tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await {
- Ok(Ok(message)) => {
- if let Some(payload) = message.payload() {
- messages.push((message.partition(), payload.to_vec()));
- }
- }
- Ok(Err(err)) => panic!("scan verifier recv error: {err}"),
- Err(_) => {}
- }
- }
- messages
-}
-
async fn wait_for(what: &str, deadline: Duration, mut condition: impl FnMut() -> bool) {
let start = Instant::now();
while !condition() {
@@ -1257,10 +1225,13 @@ async fn follower_re_establishes_consumption_from_the_committed_offset() {
.expect("incremental_assign");
let mut offsets = Vec::new();
for _ in 0..5 {
- let message = tokio::time::timeout(Duration::from_secs(15), consumer.recv())
- .await
- .expect("recv deadline")
- .expect("recv");
+ let message = tokio::time::timeout(
+ Duration::from_secs(15),
+ support::recv(&mut consumer.stream()),
+ )
+ .await
+ .expect("recv deadline")
+ .expect("recv");
offsets.push(message.offset());
}
assert_eq!(
@@ -1288,19 +1259,25 @@ async fn follower_re_establishes_consumption_from_the_committed_offset() {
.incremental_assign(&stored(&topic))
.expect("re-assign at Offset::Stored");
- let next = tokio::time::timeout(Duration::from_secs(15), consumer.recv())
- .await
- .expect("recv deadline after re-assign")
- .expect("recv after re-assign");
+ let next = tokio::time::timeout(
+ Duration::from_secs(15),
+ support::recv(&mut consumer.stream()),
+ )
+ .await
+ .expect("recv deadline after re-assign")
+ .expect("recv after re-assign");
assert_eq!(
next.offset(),
3,
"Offset::Stored must rewind to the committed offset, not keep the in-session position",
);
- let after = tokio::time::timeout(Duration::from_secs(15), consumer.recv())
- .await
- .expect("recv deadline")
- .expect("recv");
+ let after = tokio::time::timeout(
+ Duration::from_secs(15),
+ support::recv(&mut consumer.stream()),
+ )
+ .await
+ .expect("recv deadline")
+ .expect("recv");
assert_eq!(after.offset(), 4, "and redeliver the rest of the window");
})
.await;
diff --git a/rust/cohort-stream-processor/tests/merge_durability.rs b/rust/cohort-stream-processor/tests/merge_durability.rs
index 4cbf421a9ae6..2ca87d4f1202 100644
--- a/rust/cohort-stream-processor/tests/merge_durability.rs
+++ b/rust/cohort-stream-processor/tests/merge_durability.rs
@@ -19,6 +19,8 @@
// Tests seed and assert through `CohortStore` directly — the sanctioned direct-store test surface.
#![allow(clippy::disallowed_methods)]
+mod support;
+
use std::collections::HashSet;
use std::future::Future;
use std::panic::AssertUnwindSafe;
@@ -65,7 +67,6 @@ use lifecycle::{ComponentOptions, Handle, Manager};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
use rdkafka::consumer::{Consumer, StreamConsumer};
-use rdkafka::message::Message;
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::util::Timeout;
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
@@ -73,6 +74,8 @@ use serde_json::json;
use tokio::task::JoinHandle;
use uuid::Uuid;
+use support::{bootstrap_servers, consume_all};
+
const TEAM: i32 = 7;
const HASH: [u8; 16] = *b"0123456789abcdef";
const NUM_PARTITIONS: i32 = COHORT_PARTITION_COUNT as i32;
@@ -86,10 +89,6 @@ const COHORT_SINGLE: i32 = 1;
const COHORT_DAILY: i32 = 2;
const COHORT_REF: i32 = 3;
-fn bootstrap_servers() -> String {
- std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
-}
-
/// Cohort 1: `performed_event` single. Cohort 2: `performed_event_multiple gte 2` daily.
/// Cohort 3: pure cohort-ref to cohort 2, frozen with `freeze_with(UTC, true)` so it owns no
/// state-keyed leaf and can only flip via cascade from cohort 2.
@@ -316,38 +315,6 @@ fn topic_message_count(topic: &str) -> i64 {
.sum()
}
-/// Read up to `expected` messages (all partitions from the beginning). Assigned explicitly — no group
-/// join, no commit — so committed offsets the test asserts on are never perturbed.
-async fn consume_all(topic: &str, expected: usize, deadline: Duration) -> Vec<(i32, Vec<u8>)> {
- let consumer: StreamConsumer = ClientConfig::new()
- .set("bootstrap.servers", bootstrap_servers())
- .set("group.id", format!("scan-verifier-{}", Uuid::new_v4()))
- .set("enable.auto.commit", "false")
- .create()
- .expect("create scan verifier");
- let mut tpl = TopicPartitionList::new();
- for partition in 0..NUM_PARTITIONS {
- tpl.add_partition_offset(topic, partition, Offset::Beginning)
- .expect("add partition to scan TPL");
- }
- consumer.assign(&tpl).expect("assign scan verifier");
-
- let mut messages = Vec::new();
- let start = Instant::now();
- while messages.len() < expected && start.elapsed() < deadline {
- match tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await {
- Ok(Ok(message)) => {
- if let Some(payload) = message.payload() {
- messages.push((message.partition(), payload.to_vec()));
- }
- }
- Ok(Err(err)) => panic!("scan verifier recv error: {err}"),
- Err(_) => {}
- }
- }
- messages
-}
-
async fn shadow_changes(topic: &str, expected: usize) -> Vec<CohortMembershipChange> {
consume_all(topic, expected, Duration::from_secs(30))
.await
diff --git a/rust/cohort-stream-processor/tests/seed_consumer.rs b/rust/cohort-stream-processor/tests/seed_consumer.rs
index 9193c5e2211c..a28c1679326d 100644
--- a/rust/cohort-stream-processor/tests/seed_consumer.rs
+++ b/rust/cohort-stream-processor/tests/seed_consumer.rs
@@ -12,6 +12,8 @@
//! cargo test -p cohort-stream-processor --test seed_consumer -- --ignored --test-threads=1
//! ```
+mod support;
+
use std::collections::{HashMap, HashSet};
use std::future::Future;
use std::num::{NonZeroU16, NonZeroU32};
@@ -65,7 +67,6 @@ use lifecycle::{ComponentOptions, Handle, Manager};
use rdkafka::admin::{AdminClient, AdminOptions, NewTopic, TopicReplication};
use rdkafka::client::DefaultClientContext;
use rdkafka::consumer::{Consumer, ConsumerContext, StreamConsumer};
-use rdkafka::message::Message;
use rdkafka::producer::{FutureProducer, FutureRecord};
use rdkafka::util::Timeout;
use rdkafka::{ClientConfig, Offset, TopicPartitionList};
@@ -74,6 +75,8 @@ use tempfile::TempDir;
use tokio::task::JoinHandle;
use uuid::Uuid;
+use support::{bootstrap_servers, consume_all};
+
const TEAM: i32 = 7;
const COHORT: i32 = 1;
const FILTERS_HASH: &str = "0123456789abcdef";
@@ -83,10 +86,6 @@ const RECV_TIMEOUT: Duration = Duration::from_millis(200);
/// Effectively never: the idle probe must not open a fence behind an assertion's back.
const PROBE_NEVER: Duration = Duration::from_secs(3_600);
-fn bootstrap_servers() -> String {
- std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
-}
-
/// One single-leaf behavioral cohort sharing the tile's condition hash.
fn seed_catalog() -> CatalogHandle {
let leaf = json!({
@@ -249,38 +248,6 @@ fn topic_message_count(topic: &str) -> i64 {
.sum()
}
-/// Read up to `expected` messages (all partitions from the beginning). Assigned explicitly — no
-/// group join, no commit.
-async fn consume_all(topic: &str, expected: usize, deadline: Duration) -> Vec<(i32, Vec<u8>)> {
- let consumer: StreamConsumer = ClientConfig::new()
- .set("bootstrap.servers", bootstrap_servers())
- .set("group.id", format!("scan-verifier-{}", Uuid::new_v4()))
- .set("enable.auto.commit", "false")
- .create()
- .expect("create scan verifier");
- let mut tpl = TopicPartitionList::new();
- for partition in 0..NUM_PARTITIONS {
- tpl.add_partition_offset(topic, partition, Offset::Beginning)
- .expect("add partition to scan TPL");
- }
- consumer.assign(&tpl).expect("assign scan verifier");
-
- let mut messages = Vec::new();
- let start = Instant::now();
- while messages.len() < expected && start.elapsed() < deadline {
- match tokio::time::timeout(Duration::from_secs(2), consumer.recv()).await {
- Ok(Ok(message)) => {
- if let Some(payload) = message.payload() {
- messages.push((message.partition(), payload.to_vec()));
- }
- }
- Ok(Err(err)) => panic!("scan verifier recv error: {err}"),
- Err(_) => {}
- }
- }
- messages
-}
-
fn split_records(
records: Vec<(i32, Vec<u8>)>,
) -> (Vec<CohortMembershipChange>, Vec<ReconcileMarker>) {
diff --git a/rust/cohort-stream-processor/tests/support/mod.rs b/rust/cohort-stream-processor/tests/support/mod.rs
new file mode 100644
index 000000000000..d0c498f27549
--- /dev/null
+++ b/rust/cohort-stream-processor/tests/support/mod.rs
@@ -0,0 +1,68 @@
+#![allow(dead_code)]
+
+use std::time::{Duration, Instant};
+
+use cohort_stream_processor::partitions::COHORT_PARTITION_COUNT;
+use futures::{Stream, StreamExt};
+use rdkafka::consumer::{Consumer, StreamConsumer};
+use rdkafka::error::{KafkaError, KafkaResult, RDKafkaErrorCode};
+use rdkafka::message::Message;
+use rdkafka::{ClientConfig, Offset, TopicPartitionList};
+use uuid::Uuid;
+
+pub fn bootstrap_servers() -> String {
+ std::env::var("KAFKA_HOSTS").unwrap_or_else(|_| "localhost:9092".to_string())
+}
+
+pub async fn recv<S, T>(stream: &mut S) -> KafkaResult<T>
+where
+ S: Stream<Item = KafkaResult<T>> + Unpin,
+{
+ loop {
+ match stream.next().await.expect("Kafka consumer stream ended") {
+ // With a single broker, librdkafka reports `AllBrokersDown` after a dropped connection.
+ // It can also report `AllBrokersDown` alone, because it suppresses a repeated transport error.
+ Err(
+ error @ KafkaError::MessageConsumption(
+ RDKafkaErrorCode::BrokerTransportFailure | RDKafkaErrorCode::AllBrokersDown,
+ ),
+ ) => {
+ eprintln!("Kafka test reader is waiting for transport recovery: {error}");
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ result => return result,
+ }
+ }
+}
+
+/// Read up to `expected` messages (all partitions from the beginning). Assigned explicitly — no group
+/// join, no commit — so committed offsets the test asserts on are never perturbed.
+pub async fn consume_all(topic: &str, expected: usize, deadline: Duration) -> Vec<(i32, Vec<u8>)> {
+ let consumer: StreamConsumer = ClientConfig::new()
+ .set("bootstrap.servers", bootstrap_servers())
+ .set("group.id", format!("scan-verifier-{}", Uuid::new_v4()))
+ .set("enable.auto.commit", "false")
+ .create()
+ .expect("create scan verifier");
+ let mut tpl = TopicPartitionList::new();
+ for partition in 0..COHORT_PARTITION_COUNT as i32 {
+ tpl.add_partition_offset(topic, partition, Offset::Beginning)
+ .expect("add partition to scan TPL");
+ }
+ consumer.assign(&tpl).expect("assign scan verifier");
+
+ let mut messages = Vec::new();
+ let start = Instant::now();
+ while messages.len() < expected && start.elapsed() < deadline {
+ match tokio::time::timeout(Duration::from_secs(2), recv(&mut consumer.stream())).await {
+ Ok(Ok(message)) => {
+ if let Some(payload) = message.payload() {
+ messages.push((message.partition(), payload.to_vec()));
+ }
+ }
+ Ok(Err(err)) => panic!("scan verifier recv error: {err}"),
+ Err(_) => {}
+ }
+ }
+ messages
+}
diff --git a/rust/personhog-leader/tests/common/mod.rs b/rust/personhog-leader/tests/common/mod.rs
index 8b79c4b3096a..dd2b9622924b 100644
--- a/rust/personhog-leader/tests/common/mod.rs
+++ b/rust/personhog-leader/tests/common/mod.rs
@@ -54,7 +54,8 @@ use tokio_util::sync::CancellationToken;
use tonic::transport::{Channel, Server};
pub const ETCD_ENDPOINT: &str = "http://localhost:2379";
-pub const KAFKA_BOOTSTRAP: &str = "localhost:9092";
+// Not `localhost`: it also resolves to `::1`, where the broker refuses, and readers fail on that.
+pub const KAFKA_BOOTSTRAP: &str = "127.0.0.1:9092";
pub const PERSONS_DB_URL: &str = "postgres://posthog:posthog@localhost:5432/posthog_persons";
pub const WAIT_TIMEOUT: Duration = Duration::from_secs(10);
pub const POLL_INTERVAL: Duration = Duration::from_millis(100);
diff --git a/rust/personhog-leader/tests/integration.rs b/rust/personhog-leader/tests/integration.rs
index 7209764fcc34..ad57d8207ed6 100644
--- a/rust/personhog-leader/tests/integration.rs
+++ b/rust/personhog-leader/tests/integration.rs
@@ -1171,7 +1171,7 @@ async fn kafka_produce_failure_leaves_cache_unchanged() {
// ============================================================
// Test 7: E2E - update produces to local Kafka and is consumable
-// Requires local Kafka (localhost:9092) and etcd running.
+// Requires local Kafka (127.0.0.1:9092) and etcd running.
// ============================================================
#[tokio::test]