Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(personhog): retry transient broker errors in the harness kafka reader (#128533)

Zach Brownd0c9853d492c
diff --git a/rust/personhog-test-harness/src/stack/kafka.rs b/rust/personhog-test-harness/src/stack/kafka.rs
index 452a29775659..609b5694f780 100644
--- a/rust/personhog-test-harness/src/stack/kafka.rs
+++ b/rust/personhog-test-harness/src/stack/kafka.rs
@@ -209,6 +209,12 @@ fn read_to_end_blocking(
Some(Err(KafkaError::PartitionEOF(partition))) => {
remaining.remove(&partition);
}
+ // librdkafka reports a refused broker address and then reconnects on its own,
+ // for example when `localhost` resolves to `::1` first. The deadline bounds a
+ // broker that stays down.
+ Some(Err(KafkaError::MessageConsumption(
+ RDKafkaErrorCode::BrokerTransportFailure | RDKafkaErrorCode::AllBrokersDown,
+ ))) => {}
Some(Err(e)) => bail!("reading {topic}: {e}"),
Some(Ok(message)) => {
let partition = message.partition();
@@ -231,3 +237,40 @@ fn read_to_end_blocking(
}
Ok(records)
}
+
+#[cfg(test)]
+mod tests {
+ use rdkafka::producer::{BaseProducer, BaseRecord, Producer};
+ use uuid::Uuid;
+
+ use super::*;
+
+ /// Needs the CI gate's Kafka; run explicitly with `--ignored`.
+ #[tokio::test(flavor = "multi_thread")]
+ #[ignore = "needs a local Kafka"]
+ async fn read_to_end_survives_a_refused_bootstrap_address() {
+ // The broker listens on IPv4 only, so `localhost` also resolves to a refused `::1`.
+ let direct = "127.0.0.1:9092";
+ let topic = format!("personhog_harness_reader_{}", Uuid::new_v4().simple());
+ create_topic(direct, &topic, 4).await.unwrap();
+ let producer: BaseProducer = base_config(direct).create().unwrap();
+ for i in 0..100 {
+ let key = i.to_string();
+ producer
+ .send(BaseRecord::to(&topic).key(&key).payload("v"))
+ .unwrap();
+ }
+ producer.flush(Duration::from_secs(10)).unwrap();
+
+ // Each fresh reader picks an address at random, so repeat to hit `::1` first.
+ let mut failures = Vec::new();
+ for _ in 0..20 {
+ match read_to_end("localhost:9092", &topic, Duration::from_secs(20)).await {
+ Ok(records) => assert_eq!(records.len(), 100),
+ Err(e) => failures.push(e.to_string()),
+ }
+ }
+ delete_topic(direct, &topic).await.unwrap();
+ assert!(failures.is_empty(), "reads failed: {failures:?}");
+ }
+}