:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(personhog): reuse an in-flight fence preconnect instead of dialing cold (#124873)
Zach Brown9a5d997d401b
diff --git a/rust/personhog-leader/src/fencing.rs b/rust/personhog-leader/src/fencing.rsindex 10df22fcf02b..0baa41e3edc7 100644--- a/rust/personhog-leader/src/fencing.rs+++ b/rust/personhog-leader/src/fencing.rs@@ -184,12 +184,35 @@ struct Dial {broker_txn_timeout: Duration,#[cfg(any(test, feature = "test-support"))]attempts: Arc<AtomicUsize>,+ #[cfg(any(test, feature = "test-support"))]+ hold: Arc<DialHold>,+}++#[cfg(any(test, feature = "test-support"))]+#[derive(Default)]+struct DialHold {+ held: Mutex<bool>,+ released: std::sync::Condvar,}/// Connect one lane's producer, with no transactional state yet.fn connect_lane(dial: &Dial, lane: usize) -> Result<ConnectedFencedProducer, String> {#[cfg(any(test, feature = "test-support"))]dial.attempts.fetch_add(1, Ordering::SeqCst);+ #[cfg(any(test, feature = "test-support"))]+ {+ let held = dial+ .hold+ .held+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ drop(+ dial.hold+ .released+ .wait_while(held, |held| *held)+ .unwrap_or_else(PoisonError::into_inner),+ );+ }let tid = transactional_id(&dial.topic, dial.partition, lane);ConnectedFencedProducer::connect_bounded_with_context(&dial.kafka,@@ -731,6 +754,10 @@ pub struct FencedChangelogProducers {/// pod is already dying.#[cfg(any(test, feature = "test-support"))]connect_attempts: Arc<AtomicUsize>,+ #[cfg(any(test, feature = "test-support"))]+ dial_hold: Arc<DialHold>,+ #[cfg(any(test, feature = "test-support"))]+ dial_waits: AtomicUsize,/// Outcomes a test stages for the next produce on a partition.///@@ -791,6 +818,10 @@ impl FencedChangelogProducers {#[cfg(any(test, feature = "test-support"))]connect_attempts: Arc::new(AtomicUsize::new(0)),#[cfg(any(test, feature = "test-support"))]+ dial_hold: Arc::default(),+ #[cfg(any(test, feature = "test-support"))]+ dial_waits: AtomicUsize::new(0),+ #[cfg(any(test, feature = "test-support"))]staged_failures: DashMap::new(),}}@@ -815,10 +846,8 @@ impl FencedChangelogProducers {) -> Result<Arc<PartitionLanes>, String> {let topic = target.topic.as_str();let start = Instant::now();- // The removal also clears a Connecting claim: acquisition is- // happening now, so a dial still in flight is too late to help,- // and losing its claim makes it discard on completion instead- // of parking a connection nothing will consume.+ self.await_dial_in_flight(partition).await;+ // A claim still Connecting has outlived its dial; removing it discards the late result.let mut parked = self.take_prepared(partition, topic);let path = if parked.is_empty() {"cold"@@ -936,6 +965,40 @@ impl FencedChangelogProducers {broker_txn_timeout: self.broker_txn_timeout,#[cfg(any(test, feature = "test-support"))]attempts: Arc::clone(&self.connect_attempts),+ #[cfg(any(test, feature = "test-support"))]+ hold: Arc::clone(&self.dial_hold),+ }+ }++ /// Wait for the first dial seen to finish, up to its own bound, so a claim reuses its connection.+ async fn await_dial_in_flight(&self, partition: u32) {+ let mut followed = None;+ let mut waited = false;+ loop {+ let dial = self+ .prepared+ .get(&partition)+ .and_then(|slot| match slot.value() {+ Prepared::Connecting { token, since } => Some((*token, *since)),+ Prepared::Ready { .. } => None,+ });+ let first = *followed.get_or_insert(dial);+ match (dial, first) {+ (Some((token, since)), Some((first_token, _)))+ if token == first_token && since.elapsed() < self.init_timeout =>+ {+ #[cfg(any(test, feature = "test-support"))]+ if !waited {+ self.dial_waits.fetch_add(1, Ordering::SeqCst);+ }+ waited = true;+ sleep(Duration::from_millis(10)).await;+ }+ _ => break,+ }+ }+ if waited {+ counter!("personhog_leader_fence_dial_waits_total").increment(1);}}@@ -975,14 +1038,7 @@ impl FencedChangelogProducers {) -> Result<&'static str, String> {let topic = target.topic.as_str();let start = Instant::now();- while start.elapsed() < self.init_timeout- && self- .prepared- .get(&partition)- .is_some_and(|slot| matches!(slot.value(), Prepared::Connecting { .. }))- {- sleep(Duration::from_millis(10)).await;- }+ self.await_dial_in_flight(partition).await;let mut parked = self.take_prepared(partition, topic);let path = if parked.is_empty() {"cold"@@ -1219,6 +1275,30 @@ impl FencedChangelogProducers {self.connect_attempts.load(Ordering::SeqCst)}+ #[cfg(any(test, feature = "test-support"))]+ pub fn hold_dials_for_test(&self) {+ *self+ .dial_hold+ .held+ .lock()+ .unwrap_or_else(PoisonError::into_inner) = true;+ }++ #[cfg(any(test, feature = "test-support"))]+ pub fn release_dials_for_test(&self) {+ *self+ .dial_hold+ .held+ .lock()+ .unwrap_or_else(PoisonError::into_inner) = false;+ self.dial_hold.released.notify_all();+ }++ #[cfg(any(test, feature = "test-support"))]+ pub fn dial_waits_for_test(&self) -> usize {+ self.dial_waits.load(Ordering::SeqCst)+ }+/// Stage a Connecting claim of a given age, standing in for a dial/// whose task died without resolving it.#[cfg(any(test, feature = "test-support"))]diff --git a/rust/personhog-leader/tests/fencing_integration.rs b/rust/personhog-leader/tests/fencing_integration.rsindex 5f011bc29680..974acf0d1749 100644--- a/rust/personhog-leader/tests/fencing_integration.rs+++ b/rust/personhog-leader/tests/fencing_integration.rs@@ -287,6 +287,104 @@ async fn concurrent_preconnects_collapse_to_one_dial() {);}+async fn wait_until(what: &str, mut done: impl FnMut() -> bool) {+ let deadline = std::time::Instant::now() + Duration::from_secs(10);+ while !done() {+ assert!(+ std::time::Instant::now() < deadline,+ "timed out waiting for {what}"+ );+ sleep(Duration::from_millis(5)).await;+ }+}++/// An acquire during a preconnect's dial must reuse it, not open a second connection.+#[tokio::test]+async fn an_acquire_during_a_dial_reuses_its_connection() {+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());+ let producers = Arc::new(fenced_producers(1));+ producers.hold_dials_for_test();++ let dial = {+ let p = Arc::clone(&producers);+ let t = target(&topic);+ tokio::spawn(async move { p.preconnect(4, &t).await })+ };+ wait_until("the preconnect to start dialing", || {+ producers.connect_attempts_for_test() == 1+ })+ .await;+ let acquire = {+ let p = Arc::clone(&producers);+ let t = target(&topic);+ tokio::spawn(async move { p.acquire(4, &t).await })+ };+ // Either the acquire waits for the held dial, or it dials its own.+ wait_until("the acquire to wait or dial", || {+ producers.dial_waits_for_test() == 1 || producers.connect_attempts_for_test() == 2+ })+ .await;+ producers.release_dials_for_test();++ dial.await.unwrap();+ acquire.await.unwrap().expect("the acquire succeeds");+ assert_eq!(+ producers.connect_attempts_for_test(),+ 1,+ "the acquire must reuse the dial in flight, not open a second connection"+ );+}++/// The wait ends at the dial's own bound, not a full timeout from the acquire.+#[tokio::test]+async fn an_acquire_waits_only_for_what_the_dial_has_left() {+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());+ // The helper's 10s init timeout bounds the dial.+ let producers = fenced_producers(1);+ producers.stage_connecting_for_test(2, Duration::from_millis(9_500));++ let start = std::time::Instant::now();+ producers+ .acquire(2, &target(&topic))+ .await+ .expect("the acquire goes cold once the dial's bound passes");+ assert!(+ start.elapsed() < Duration::from_secs(5),+ "waited {:?}; the wait must end at the dial's bound, not a full timeout",+ start.elapsed()+ );+}++/// The wait follows only the first dial, so replacement dials cannot hold the acquire.+#[tokio::test]+async fn an_acquire_does_not_wait_out_a_replacement_dial() {+ let topic = format!("fence_prepared_{}", uuid::Uuid::new_v4().simple());+ let producers = Arc::new(fenced_producers(1));+ producers.stage_connecting_for_test(3, Duration::from_millis(9_000));++ let start = std::time::Instant::now();+ let acquire = {+ let p = Arc::clone(&producers);+ let t = target(&topic);+ tokio::spawn(async move { p.acquire(3, &t).await })+ };+ wait_until("the acquire to wait on the first dial", || {+ producers.dial_waits_for_test() == 1+ })+ .await;+ producers.stage_connecting_for_test(3, Duration::ZERO);++ acquire+ .await+ .unwrap()+ .expect("the acquire goes cold once the dial it followed is gone");+ assert!(+ start.elapsed() < Duration::from_secs(5),+ "waited {:?}; the acquire must not follow the replacement dial",+ start.elapsed()+ );+}+/// A failed dial must release its claim: the claim exists to suppress/// concurrent dials, not to wedge the partition out of preconnecting/// after a broker hiccup.