Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): reuse an in-flight fence preconnect instead of dialing cold (#124873)

Zach Brown9a5d997d401b
diff --git a/rust/personhog-leader/src/fencing.rs b/rust/personhog-leader/src/fencing.rs
index 10df22fcf02b..0baa41e3edc7 100644
--- a/rust/personhog-leader/src/fencing.rs
+++ b/rust/personhog-leader/src/fencing.rs
@@ -184,12 +184,35 @@ struct Dial {
broker_txn_timeout: Duration,
#[cfg(any(test, feature = "test-support"))]
attempts: Arc<AtomicUsize>,
+ #[cfg(any(test, feature = "test-support"))]
+ hold: Arc<DialHold>,
+}
+
+#[cfg(any(test, feature = "test-support"))]
+#[derive(Default)]
+struct DialHold {
+ held: Mutex<bool>,
+ released: std::sync::Condvar,
}
/// Connect one lane's producer, with no transactional state yet.
fn connect_lane(dial: &Dial, lane: usize) -> Result<ConnectedFencedProducer, String> {
#[cfg(any(test, feature = "test-support"))]
dial.attempts.fetch_add(1, Ordering::SeqCst);
+ #[cfg(any(test, feature = "test-support"))]
+ {
+ let held = dial
+ .hold
+ .held
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner);
+ drop(
+ dial.hold
+ .released
+ .wait_while(held, |held| *held)
+ .unwrap_or_else(PoisonError::into_inner),
+ );
+ }
let tid = transactional_id(&dial.topic, dial.partition, lane);
ConnectedFencedProducer::connect_bounded_with_context(
&dial.kafka,
@@ -731,6 +754,10 @@ pub struct FencedChangelogProducers {
/// pod is already dying.
#[cfg(any(test, feature = "test-support"))]
connect_attempts: Arc<AtomicUsize>,
+ #[cfg(any(test, feature = "test-support"))]
+ dial_hold: Arc<DialHold>,
+ #[cfg(any(test, feature = "test-support"))]
+ dial_waits: AtomicUsize,
/// Outcomes a test stages for the next produce on a partition.
///
@@ -791,6 +818,10 @@ impl FencedChangelogProducers {
#[cfg(any(test, feature = "test-support"))]
connect_attempts: Arc::new(AtomicUsize::new(0)),
#[cfg(any(test, feature = "test-support"))]
+ dial_hold: Arc::default(),
+ #[cfg(any(test, feature = "test-support"))]
+ dial_waits: AtomicUsize::new(0),
+ #[cfg(any(test, feature = "test-support"))]
staged_failures: DashMap::new(),
}
}
@@ -815,10 +846,8 @@ impl FencedChangelogProducers {
) -> Result<Arc<PartitionLanes>, String> {
let topic = target.topic.as_str();
let start = Instant::now();
- // The removal also clears a Connecting claim: acquisition is
- // happening now, so a dial still in flight is too late to help,
- // and losing its claim makes it discard on completion instead
- // of parking a connection nothing will consume.
+ self.await_dial_in_flight(partition).await;
+ // A claim still Connecting has outlived its dial; removing it discards the late result.
let mut parked = self.take_prepared(partition, topic);
let path = if parked.is_empty() {
"cold"
@@ -936,6 +965,40 @@ impl FencedChangelogProducers {
broker_txn_timeout: self.broker_txn_timeout,
#[cfg(any(test, feature = "test-support"))]
attempts: Arc::clone(&self.connect_attempts),
+ #[cfg(any(test, feature = "test-support"))]
+ hold: Arc::clone(&self.dial_hold),
+ }
+ }
+
+ /// Wait for the first dial seen to finish, up to its own bound, so a claim reuses its connection.
+ async fn await_dial_in_flight(&self, partition: u32) {
+ let mut followed = None;
+ let mut waited = false;
+ loop {
+ let dial = self
+ .prepared
+ .get(&partition)
+ .and_then(|slot| match slot.value() {
+ Prepared::Connecting { token, since } => Some((*token, *since)),
+ Prepared::Ready { .. } => None,
+ });
+ let first = *followed.get_or_insert(dial);
+ match (dial, first) {
+ (Some((token, since)), Some((first_token, _)))
+ if token == first_token && since.elapsed() < self.init_timeout =>
+ {
+ #[cfg(any(test, feature = "test-support"))]
+ if !waited {
+ self.dial_waits.fetch_add(1, Ordering::SeqCst);
+ }
+ waited = true;
+ sleep(Duration::from_millis(10)).await;
+ }
+ _ => break,
+ }
+ }
+ if waited {
+ counter!("personhog_leader_fence_dial_waits_total").increment(1);
}
}
@@ -975,14 +1038,7 @@ impl FencedChangelogProducers {
) -> Result<&'static str, String> {
let topic = target.topic.as_str();
let start = Instant::now();
- while start.elapsed() < self.init_timeout
- && self
- .prepared
- .get(&partition)
- .is_some_and(|slot| matches!(slot.value(), Prepared::Connecting { .. }))
- {
- sleep(Duration::from_millis(10)).await;
- }
+ self.await_dial_in_flight(partition).await;
let mut parked = self.take_prepared(partition, topic);
let path = if parked.is_empty() {
"cold"
@@ -1219,6 +1275,30 @@ impl FencedChangelogProducers {
self.connect_attempts.load(Ordering::SeqCst)
}
+ #[cfg(any(test, feature = "test-support"))]
+ pub fn hold_dials_for_test(&self) {
+ *self
+ .dial_hold
+ .held
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner) = true;
+ }
+
+ #[cfg(any(test, feature = "test-support"))]
+ pub fn release_dials_for_test(&self) {
+ *self
+ .dial_hold
+ .held
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner) = false;
+ self.dial_hold.released.notify_all();
+ }
+
+ #[cfg(any(test, feature = "test-support"))]
+ pub fn dial_waits_for_test(&self) -> usize {
+ self.dial_waits.load(Ordering::SeqCst)
+ }
+
/// Stage a Connecting claim of a given age, standing in for a dial
/// whose task died without resolving it.
#[cfg(any(test, feature = "test-support"))]
diff --git a/rust/personhog-leader/tests/fencing_integration.rs b/rust/personhog-leader/tests/fencing_integration.rs
index 5f011bc29680..974acf0d1749 100644
--- a/rust/personhog-leader/tests/fencing_integration.rs
+++ b/rust/personhog-leader/tests/fencing_integration.rs
@@ -287,6 +287,104 @@ async fn concurrent_preconnects_collapse_to_one_dial() {
);
}
+async fn wait_until(what: &str, mut done: impl FnMut() -> bool) {
+ let deadline = std::time::Instant::now() + Duration::from_secs(10);
+ while !done() {
+ assert!(
+ std::time::Instant::now() < deadline,
+ "timed out waiting for {what}"
+ );
+ sleep(Duration::from_millis(5)).await;
+ }
+}
+
+/// An acquire during a preconnect's dial must reuse it, not open a second connection.
+#[tokio::test]
+async fn an_acquire_during_a_dial_reuses_its_connection() {
+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());
+ let producers = Arc::new(fenced_producers(1));
+ producers.hold_dials_for_test();
+
+ let dial = {
+ let p = Arc::clone(&producers);
+ let t = target(&topic);
+ tokio::spawn(async move { p.preconnect(4, &t).await })
+ };
+ wait_until("the preconnect to start dialing", || {
+ producers.connect_attempts_for_test() == 1
+ })
+ .await;
+ let acquire = {
+ let p = Arc::clone(&producers);
+ let t = target(&topic);
+ tokio::spawn(async move { p.acquire(4, &t).await })
+ };
+ // Either the acquire waits for the held dial, or it dials its own.
+ wait_until("the acquire to wait or dial", || {
+ producers.dial_waits_for_test() == 1 || producers.connect_attempts_for_test() == 2
+ })
+ .await;
+ producers.release_dials_for_test();
+
+ dial.await.unwrap();
+ acquire.await.unwrap().expect("the acquire succeeds");
+ assert_eq!(
+ producers.connect_attempts_for_test(),
+ 1,
+ "the acquire must reuse the dial in flight, not open a second connection"
+ );
+}
+
+/// The wait ends at the dial's own bound, not a full timeout from the acquire.
+#[tokio::test]
+async fn an_acquire_waits_only_for_what_the_dial_has_left() {
+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());
+ // The helper's 10s init timeout bounds the dial.
+ let producers = fenced_producers(1);
+ producers.stage_connecting_for_test(2, Duration::from_millis(9_500));
+
+ let start = std::time::Instant::now();
+ producers
+ .acquire(2, &target(&topic))
+ .await
+ .expect("the acquire goes cold once the dial's bound passes");
+ assert!(
+ start.elapsed() < Duration::from_secs(5),
+ "waited {:?}; the wait must end at the dial's bound, not a full timeout",
+ start.elapsed()
+ );
+}
+
+/// The wait follows only the first dial, so replacement dials cannot hold the acquire.
+#[tokio::test]
+async fn an_acquire_does_not_wait_out_a_replacement_dial() {
+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());
+ let producers = Arc::new(fenced_producers(1));
+ producers.stage_connecting_for_test(3, Duration::from_millis(9_000));
+
+ let start = std::time::Instant::now();
+ let acquire = {
+ let p = Arc::clone(&producers);
+ let t = target(&topic);
+ tokio::spawn(async move { p.acquire(3, &t).await })
+ };
+ wait_until("the acquire to wait on the first dial", || {
+ producers.dial_waits_for_test() == 1
+ })
+ .await;
+ producers.stage_connecting_for_test(3, Duration::ZERO);
+
+ acquire
+ .await
+ .unwrap()
+ .expect("the acquire goes cold once the dial it followed is gone");
+ assert!(
+ start.elapsed() < Duration::from_secs(5),
+ "waited {:?}; the acquire must not follow the replacement dial",
+ start.elapsed()
+ );
+}
+
/// A failed dial must release its claim: the claim exists to suppress
/// concurrent dials, not to wedge the partition out of preconnecting
/// after a broker hiccup.