:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): switch etcd members when the current one cannot serve (#123639)
Zach Brown8e699ae37aa5
diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 52c49e18bf24..b770728b17ec 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -552,12 +552,14 @@ name = "assignment-coordination"version = "0.1.0"dependencies = ["etcd-client",+ "h2 0.4.13","metrics","serde","serde_json","thiserror 2.0.18","tokio","tokio-util",+ "tonic 0.14.6","tracing",]diff --git a/rust/common/assignment-coordination/Cargo.toml b/rust/common/assignment-coordination/Cargo.tomlindex e3a5aa8e0930..f43e05ea3a7c 100644--- a/rust/common/assignment-coordination/Cargo.toml+++ b/rust/common/assignment-coordination/Cargo.toml@@ -6,12 +6,14 @@ publish = false[dependencies]etcd-client = "0.18"+h2 = "0.4"metrics = { workspace = true }serde = { workspace = true }serde_json = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true }tokio-util = { version = "0.7", features = ["rt"] }+tonic = { version = "0.14", default-features = false, features = ["transport"] }tracing = { workspace = true }[lints]diff --git a/rust/common/assignment-coordination/src/failover.rs b/rust/common/assignment-coordination/src/failover.rsnew file mode 100644index 000000000000..13c1a050b16c--- /dev/null+++ b/rust/common/assignment-coordination/src/failover.rs@@ -0,0 +1,374 @@+use std::collections::hash_map::RandomState;+use std::hash::{BuildHasher, Hasher};+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++use etcd_client::{Client, ConnectOptions, GetOptions};+use tonic::Code;++/// Consecutive member-level failures that start a switch attempt.+const FAILURE_THRESHOLD: u32 = 3;+/// Minimum gap between switch attempts, so a cluster-wide outage does not+/// turn every failed call into a round of probes.+const MIN_ATTEMPT_INTERVAL: Duration = Duration::from_secs(5);+/// Budget for one probe: connect, one linearizable read, one member list.+const PROBE_TIMEOUT: Duration = Duration::from_secs(3);+/// An unanswered call that waited at least this long starts a switch at once.+/// Shorter waits, such as the coordinator's 1s renewals on a busy cluster,+/// count toward FAILURE_THRESHOLD instead.+const UNANSWERED_FLOOR: Duration = Duration::from_secs(2);++/// Whether an etcd error says this member cannot serve: Unavailable,+/// DeadlineExceeded, or a transport failure under the call. Errors about the+/// request itself (NOSPACE, a failed compare, a compacted revision) do not.+pub fn is_member_level(err: &etcd_client::Error) -> bool {+ match err {+ etcd_client::Error::GRpcStatus(status) => {+ matches!(status.code(), Code::Unavailable | Code::DeadlineExceeded)+ || raised_by_connection(status)+ }+ etcd_client::Error::TransportError(_) | etcd_client::Error::IoError(_) => true,+ _ => false,+ }+}++/// Whether the client raised this status for a failed connection or HTTP/2+/// stream, whatever its code, rather than etcd sending it.+fn raised_by_connection(status: &tonic::Status) -> bool {+ let mut source = std::error::Error::source(status);+ while let Some(err) = source {+ if err.is::<tonic::transport::Error>()+ || err.is::<h2::Error>()+ || err.is::<std::io::Error>()+ {+ return true;+ }+ source = err.source();+ }+ false+}++struct State {+ /// Endpoints currently in the client's balanced channel.+ current: Vec<String>,+ /// The endpoints the store was configured with, used when no member+ /// list is known.+ configured: Vec<String>,+ /// One client URL per voting member, from the last member list read.+ members: Vec<String>,+ last_attempt: Option<Instant>,+ last_failed_refresh: Option<Instant>,+}++/// Clears a flag when dropped, so a task that panics cannot leave it set.+struct ClearOnDrop(Arc<AtomicBool>);++impl Drop for ClearOnDrop {+ fn drop(&mut self) {+ self.0.store(false, Ordering::Release);+ }+}++/// Moves a client to a different etcd member when the one it is connected+/// to keeps failing while the rest of the cluster may still be healthy.+pub(crate) struct MemberFailover {+ client: Client,+ probe_options: ConnectOptions,+ consecutive_failures: AtomicU32,+ switching: Arc<AtomicBool>,+ refreshing: Arc<AtomicBool>,+ members_known: AtomicBool,+ state: Mutex<State>,+}++impl MemberFailover {+ pub(crate) fn new(client: Client, options: ConnectOptions, endpoints: Vec<String>) -> Self {+ Self {+ client,+ probe_options: options+ .with_require_leader(true)+ .with_connect_timeout(PROBE_TIMEOUT),+ consecutive_failures: AtomicU32::new(0),+ switching: Arc::new(AtomicBool::new(false)),+ refreshing: Arc::new(AtomicBool::new(false)),+ members_known: AtomicBool::new(false),+ state: Mutex::new(State {+ current: endpoints.clone(),+ configured: endpoints,+ members: Vec::new(),+ last_attempt: None,+ last_failed_refresh: None,+ }),+ }+ }++ /// Reads the member list through the connected client, best effort.+ pub(crate) async fn refresh_members(&self) {+ let mut client = self.client.clone();+ let members = match tokio::time::timeout(PROBE_TIMEOUT, client.member_list()).await {+ Ok(Ok(resp)) => member_urls(&resp),+ _ => Vec::new(),+ };+ if members.is_empty() {+ self.lock().last_failed_refresh = Some(Instant::now());+ } else {+ self.set_members(members);+ }+ }++ pub(crate) fn note_success(self: &Arc<Self>) {+ self.consecutive_failures.store(0, Ordering::Relaxed);+ if !self.members_known.load(Ordering::Acquire) {+ self.start_refresh();+ }+ }++ /// Reads the member list in the background once a call succeeds, for a+ /// store that started while etcd could not answer.+ fn start_refresh(self: &Arc<Self>) {+ if self.refreshing.swap(true, Ordering::AcqRel) {+ return;+ }+ let latch = ClearOnDrop(Arc::clone(&self.refreshing));+ if self+ .lock()+ .last_failed_refresh+ .is_some_and(|at| at.elapsed() < MIN_ATTEMPT_INTERVAL)+ {+ return;+ }+ let this = Arc::clone(self);+ tokio::spawn(async move {+ let _latch = latch;+ this.refresh_members().await;+ });+ }++ pub(crate) fn note_failure(self: &Arc<Self>) {+ let failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed) + 1;+ if failures >= FAILURE_THRESHOLD {+ self.start_switch();+ }+ }++ /// A call that got no answer after `waited`.+ pub(crate) fn note_unanswered(self: &Arc<Self>, waited: Duration) {+ if waited < UNANSWERED_FLOOR {+ self.note_failure();+ return;+ }+ self.consecutive_failures.fetch_add(1, Ordering::Relaxed);+ self.start_switch();+ }++ fn start_switch(self: &Arc<Self>) {+ if self.switching.swap(true, Ordering::AcqRel) {+ return;+ }+ let latch = ClearOnDrop(Arc::clone(&self.switching));+ let this = Arc::clone(self);+ tokio::spawn(async move {+ let _latch = latch;+ this.attempt_switch().await;+ });+ }++ async fn attempt_switch(&self) {+ let (current, candidates) = {+ let mut state = self.lock();+ if state+ .last_attempt+ .is_some_and(|at| at.elapsed() < MIN_ATTEMPT_INTERVAL)+ {+ return;+ }+ state.last_attempt = Some(Instant::now());+ let candidates: Vec<String> = state+ .members+ .iter()+ .filter(|url| !state.current.contains(url))+ .cloned()+ .collect();+ (state.current.clone(), candidates)+ };++ if candidates.is_empty() {+ // No other member is known: reconnect so the configured names+ // resolve again.+ let configured = self.lock().configured.clone();+ tracing::warn!(+ endpoints = ?configured,+ "etcd member keeps failing and no other member is known; reconnecting"+ );+ let outcome = match self.replace_endpoints(¤t, &configured).await {+ Ok(()) => "reconnected",+ Err(()) => "swap_failed",+ };+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => outcome)+ .increment(1);+ return;+ }++ for candidate in rotated(candidates) {+ match self.probe(&candidate).await {+ Ok(members) => {+ if self+ .replace_endpoints(¤t, std::slice::from_ref(&candidate))+ .await+ .is_err()+ {+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "swap_failed")+ .increment(1);+ return;+ }+ if !members.is_empty() {+ self.set_members(members);+ }+ self.consecutive_failures.store(0, Ordering::Relaxed);+ tracing::warn!(+ from = ?current,+ to = %candidate,+ "switched etcd member after repeated failures"+ );+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "switched")+ .increment(1);+ return;+ }+ Err(error) => {+ tracing::warn!(%candidate, %error, "etcd member probe failed");+ }+ }+ }+ tracing::warn!(+ current = ?current,+ "etcd member keeps failing and no other member answered a probe; staying"+ );+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "no_healthy_member")+ .increment(1);+ }++ /// Connects to `url` alone and checks it can serve a linearizable read,+ /// which needs a leader and a quorum. Returns its view of the members.+ async fn probe(&self, url: &str) -> Result<Vec<String>, String> {+ let attempt = async {+ let mut client = Client::connect([url], Some(self.probe_options.clone())).await?;+ client+ .get("probe", Some(GetOptions::new().with_count_only()))+ .await?;+ let members = client.member_list().await?;+ Ok::<_, etcd_client::Error>(member_urls(&members))+ };+ match tokio::time::timeout(PROBE_TIMEOUT, attempt).await {+ Ok(Ok(members)) => Ok(members),+ Ok(Err(error)) => Err(error.to_string()),+ Err(_) => Err(format!("no answer within {PROBE_TIMEOUT:?}")),+ }+ }++ /// Inserts the new endpoints before removing the old ones, so the+ /// balanced channel is never empty. Fails, leaving the old endpoints in+ /// place, when an insert fails.+ async fn replace_endpoints(&self, old: &[String], new: &[String]) -> Result<(), ()> {+ for url in new {+ if let Err(error) = self.client.add_endpoint(url).await {+ tracing::error!(%url, %error, "failed to add etcd endpoint; keeping the current one");+ return Err(());+ }+ }+ let mut current = new.to_vec();+ for url in old.iter().filter(|url| !new.contains(url)) {+ if let Err(error) = self.client.remove_endpoint(url).await {+ tracing::error!(%url, %error, "failed to remove etcd endpoint");+ current.push(url.clone());+ }+ }+ self.lock().current = current;+ Ok(())+ }++ fn set_members(&self, members: Vec<String>) {+ self.members_known+ .store(!members.is_empty(), Ordering::Release);+ self.lock().members = members;+ }++ fn lock(&self) -> std::sync::MutexGuard<'_, State> {+ self.state+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ }++ #[cfg(test)]+ pub(crate) fn set_members_for_test(&self, members: Vec<String>) {+ self.set_members(members);+ }++ #[cfg(test)]+ pub(crate) fn current_for_test(&self) -> Vec<String> {+ self.lock().current.clone()+ }++ #[cfg(test)]+ pub(crate) fn members_for_test(&self) -> Vec<String> {+ self.lock().members.clone()+ }++ #[cfg(test)]+ pub(crate) fn switching_for_test(&self) -> bool {+ self.switching.load(Ordering::Acquire)+ }+}++fn member_urls(resp: &etcd_client::MemberListResponse) -> Vec<String> {+ resp.members()+ .iter()+ .filter(|member| !member.is_learner())+ .filter_map(|member| member.client_urls().first().cloned())+ .collect()+}++/// Starts the candidate list at a random member so processes leaving the+/// same member spread across the others.+fn rotated(mut candidates: Vec<String>) -> Vec<String> {+ let mut hasher = RandomState::new().build_hasher();+ hasher.write_u64(0);+ let start = (hasher.finish() as usize) % candidates.len();+ candidates.rotate_left(start);+ candidates+}++#[cfg(test)]+mod tests {+ use super::*;++ fn status(code: Code) -> etcd_client::Error {+ etcd_client::Error::GRpcStatus(tonic::Status::new(code, "test"))+ }++ #[test]+ fn member_level_errors_are_the_ones_a_different_member_could_avoid() {+ assert!(is_member_level(&status(Code::Unavailable)));+ assert!(is_member_level(&status(Code::DeadlineExceeded)));+ assert!(is_member_level(&etcd_client::Error::IoError(+ std::io::Error::from(std::io::ErrorKind::ConnectionReset)+ )));+ let broken_stream =+ tonic::Status::from_error(Box::new(h2::Error::from(h2::Reason::STREAM_CLOSED)));+ assert_eq!(broken_stream.code(), Code::Unknown);+ assert!(is_member_level(&etcd_client::Error::GRpcStatus(+ broken_stream+ )));+ for code in [+ Code::Unknown,+ Code::ResourceExhausted,+ Code::FailedPrecondition,+ Code::NotFound,+ Code::InvalidArgument,+ Code::OutOfRange,+ Code::PermissionDenied,+ ] {+ assert!(!is_member_level(&status(code)), "{code:?}");+ }+ }+}diff --git a/rust/common/assignment-coordination/src/lib.rs b/rust/common/assignment-coordination/src/lib.rsindex 0dedaa1b60a9..8f276262df95 100644--- a/rust/common/assignment-coordination/src/lib.rs+++ b/rust/common/assignment-coordination/src/lib.rs@@ -1,4 +1,5 @@pub mod error;+pub mod failover;pub mod leader_election;pub mod store;pub mod strategy;diff --git a/rust/common/assignment-coordination/src/store.rs b/rust/common/assignment-coordination/src/store.rsindex 569be9056096..87a110fd18cf 100644--- a/rust/common/assignment-coordination/src/store.rs+++ b/rust/common/assignment-coordination/src/store.rs@@ -1,3 +1,4 @@+use std::sync::Arc;use std::time::Duration;use etcd_client::{@@ -8,6 +9,7 @@ use serde::de::DeserializeOwned;use serde::Serialize;use crate::error::{Error, Result};+use crate::failover::{is_member_level, MemberFailover};#[derive(Debug, Clone)]pub struct StoreConfig {@@ -27,6 +29,7 @@ pub struct StoreConfig {pub struct EtcdStore {client: Client,config: StoreConfig,+ failover: Option<Arc<MemberFailover>>,}/// Records one etcd operation's wall time on drop, so every return path@@ -68,39 +71,94 @@ fn record_payload_bytes(op: &'static str, bytes: usize) {.record(bytes as f64);}+// HTTP/2 pings fail a silent connection in about 15s. A per-request timeout+// would also end healthy watch streams, so there is none. The interval stays+// above etcd's 5s keepalive floor, and idle pings stay off because etcd+// counts them as strikes.+fn connect_options() -> ConnectOptions {+ ConnectOptions::new()+ .with_connect_timeout(Duration::from_secs(5))+ .with_keep_alive(Duration::from_secs(10), Duration::from_secs(5))+ .with_keep_alive_while_idle(false)+}+fn kvs_bytes(kvs: &[etcd_client::KeyValue]) -> usize {kvs.iter().map(|kv| kv.key().len() + kv.value().len()).sum()}impl EtcdStore {pub async fn connect(config: StoreConfig) -> Result<Self> {- // Transport-level liveness so a silent network partition fails- // fast instead of hanging until TCP retransmission gives up- // (minutes — far past any lease TTL): HTTP/2 pings ride every- // connection, including idle ones and long-lived watch streams,- // and error all in-flight requests within roughly one ping- // interval plus its timeout of the peer going dark. Deliberately- // no per-request timeout — it would apply to the whole lifetime- // of a watch stream and kill healthy watches.- // The ping interval must clear etcd's server-side gRPC keepalive- // enforcement (--grpc-keepalive-min-time, default 5s): pings at or- // under the floor are strikes, and two strikes close the- // connection with GOAWAY. Idle pings are likewise strikes unless- // the server permits them, so they stay off — every component- // that matters holds an active watch or keepalive stream, and an- // idle channel is revalidated on next use within the connect- // timeout.- let options = ConnectOptions::new()- .with_connect_timeout(Duration::from_secs(5))- .with_keep_alive(Duration::from_secs(10), Duration::from_secs(5))- .with_keep_alive_while_idle(false);- let client = Client::connect(&config.endpoints, Some(options)).await?;- Ok(Self { client, config })+ let client = Client::connect(&config.endpoints, Some(connect_options())).await?;+ Ok(Self {+ client,+ config,+ failover: None,+ })+ }++ /// Like `connect`, and moves to another member when this one keeps+ /// failing. Requests carry etcd's require-leader flag, so a member+ /// without a leader rejects them at once.+ pub async fn connect_with_member_failover(config: StoreConfig) -> Result<Self> {+ let options = connect_options().with_require_leader(true);+ let client = Client::connect(&config.endpoints, Some(options.clone())).await?;+ let failover = Arc::new(MemberFailover::new(+ client.clone(),+ options,+ config.endpoints.clone(),+ ));+ failover.refresh_members().await;+ Ok(Self {+ client,+ config,+ failover: Some(failover),+ })}/// Shares an existing client's channel instead of opening another.pub fn from_client(client: Client, config: StoreConfig) -> Self {- Self { client, config }+ Self {+ client,+ config,+ failover: None,+ }+ }++ /// Records a call made outside this store's methods, such as a lease+ /// renewal on a `LeaseKeeper`, so it counts toward member failover.+ pub fn observe_member_outcome(&self, outcome: std::result::Result<(), &etcd_client::Error>) {+ let Some(failover) = &self.failover else {+ return;+ };+ match outcome {+ Ok(()) => failover.note_success(),+ Err(err) if is_member_level(err) => failover.note_failure(),+ Err(_) => {}+ }+ }++ /// Records a call that got no answer within `waited`, such as a lease+ /// renewal. A long enough wait starts a switch attempt at once.+ pub fn observe_member_unanswered(&self, waited: Duration) {+ if let Some(failover) = &self.failover {+ failover.note_unanswered(waited);+ }+ }++ /// Records a call outside this store that failed without an etcd error,+ /// such as a keepalive stream that closed.+ pub fn observe_member_failure(&self) {+ if let Some(failover) = &self.failover {+ failover.note_failure();+ }+ }++ fn observe<T>(+ &self,+ result: std::result::Result<T, etcd_client::Error>,+ ) -> std::result::Result<T, etcd_client::Error> {+ self.observe_member_outcome(result.as_ref().map(|_| ()));+ result}pub fn prefix(&self) -> &str {@@ -115,7 +173,7 @@ impl EtcdStore {pub async fn get_raw(&self, key: &str) -> Result<Option<Vec<u8>>> {let _t = OpTimer::new("get_raw");- let resp = self.client.clone().get(key, None).await?;+ let resp = self.observe(self.client.clone().get(key, None).await)?;record_payload_bytes("get_raw", kvs_bytes(resp.kvs()));Ok(resp.kvs().first().map(|kv| kv.value().to_vec()))}@@ -124,7 +182,7 @@ impl EtcdStore {let _t = OpTimer::new("put_raw");let value = value.into();record_payload_bytes("put_raw", key.len() + value.len());- self.client.clone().put(key, value, None).await?;+ self.observe(self.client.clone().put(key, value, None).await)?;Ok(())}@@ -132,7 +190,7 @@ impl EtcdStore {pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {let _t = OpTimer::new("get");- let resp = self.client.clone().get(key, None).await?;+ let resp = self.observe(self.client.clone().get(key, None).await)?;record_payload_bytes("get", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => Ok(Some(serde_json::from_slice(kv.value())?)),@@ -142,7 +200,7 @@ impl EtcdStore {pub async fn get_versioned<T: DeserializeOwned>(&self, key: &str) -> Result<Option<(T, i64)>> {let _t = OpTimer::new("get_versioned");- let resp = self.client.clone().get(key, None).await?;+ let resp = self.observe(self.client.clone().get(key, None).await)?;record_payload_bytes("get_versioned", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => {@@ -164,7 +222,7 @@ impl EtcdStore {key: &str,) -> Result<Option<(T, i64)>> {let _t = OpTimer::new("get_with_mod_revision");- let resp = self.client.clone().get(key, None).await?;+ let resp = self.observe(self.client.clone().get(key, None).await)?;record_payload_bytes("get_with_mod_revision", kvs_bytes(resp.kvs()));match resp.kvs().first() {Some(kv) => {@@ -185,7 +243,7 @@ impl EtcdStore {key: &str,) -> Result<(Option<T>, i64)> {let _t = OpTimer::new("get_with_revision");- let resp = self.client.clone().get(key, None).await?;+ let resp = self.observe(self.client.clone().get(key, None).await)?;record_payload_bytes("get_with_revision", kvs_bytes(resp.kvs()));let revision = resp.header().map(|h| h.revision()).unwrap_or(0);match resp.kvs().first() {@@ -205,7 +263,7 @@ impl EtcdStore {pub async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {let _t = OpTimer::new("list_keys");let options = GetOptions::new().with_prefix().with_keys_only();- let resp = self.client.clone().get(prefix, Some(options)).await?;+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;record_payload_bytes("list_keys", kvs_bytes(resp.kvs()));Ok(resp.kvs()@@ -219,7 +277,7 @@ impl EtcdStore {pub async fn list_keys_with_mod_revisions(&self, prefix: &str) -> Result<Vec<(String, i64)>> {let _t = OpTimer::new("list_keys_with_mod_revisions");let options = GetOptions::new().with_prefix().with_keys_only();- let resp = self.client.clone().get(prefix, Some(options)).await?;+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;record_payload_bytes("list_keys_with_mod_revisions", kvs_bytes(resp.kvs()));Ok(resp.kvs()@@ -243,7 +301,7 @@ impl EtcdStore {) -> Result<(Vec<T>, i64)> {let _t = OpTimer::new("list_with_revision");let options = GetOptions::new().with_prefix();- let resp = self.client.clone().get(prefix, Some(options)).await?;+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;record_payload_bytes("list_with_revision", kvs_bytes(resp.kvs()));let revision = resp.header().map(|h| h.revision()).unwrap_or(0);let items = resp@@ -263,7 +321,7 @@ impl EtcdStore {) -> Result<Vec<(T, i64)>> {let _t = OpTimer::new("list_with_mod_revisions");let options = GetOptions::new().with_prefix();- let resp = self.client.clone().get(prefix, Some(options)).await?;+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;record_payload_bytes("list_with_mod_revisions", kvs_bytes(resp.kvs()));resp.kvs().iter()@@ -294,27 +352,27 @@ impl EtcdStore {let value = serde_json::to_string(value)?;record_payload_bytes("put", key.len() + value.len());let options = lease_id.map(|id| PutOptions::new().with_lease(id));- self.client.clone().put(key, value, options).await?;+ self.observe(self.client.clone().put(key, value, options).await)?;Ok(())}pub async fn delete(&self, key: &str) -> Result<()> {let _t = OpTimer::new("delete");- self.client.clone().delete(key, None).await?;+ self.observe(self.client.clone().delete(key, None).await)?;Ok(())}pub async fn delete_prefix(&self, prefix: &str) -> Result<()> {let _t = OpTimer::new("delete_prefix");let options = DeleteOptions::new().with_prefix();- self.client.clone().delete(prefix, Some(options)).await?;+ self.observe(self.client.clone().delete(prefix, Some(options)).await)?;Ok(())}pub async fn watch(&self, prefix: &str) -> Result<WatchStream> {let _t = OpTimer::new("watch");let options = WatchOptions::new().with_prefix();- let stream = self.client.clone().watch(prefix, Some(options)).await?;+ let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;Ok(stream)}@@ -330,7 +388,7 @@ impl EtcdStore {let options = WatchOptions::new().with_prefix().with_start_revision(start_revision);- let stream = self.client.clone().watch(prefix, Some(options)).await?;+ let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;Ok(stream)}@@ -340,7 +398,7 @@ impl EtcdStore {pub async fn watch_key_from(&self, key: &str, start_revision: i64) -> Result<WatchStream> {let _t = OpTimer::new("watch_key_from");let options = WatchOptions::new().with_start_revision(start_revision);- let stream = self.client.clone().watch(key, Some(options)).await?;+ let stream = self.observe(self.client.clone().watch(key, Some(options)).await)?;Ok(stream)}@@ -355,7 +413,7 @@ impl EtcdStore {/// request side directly instead.pub async fn txn(&self, txn: Txn) -> Result<TxnResponse> {let _t = OpTimer::new("txn");- Ok(self.client.clone().txn(txn).await?)+ Ok(self.observe(self.client.clone().txn(txn).await)?)}/// Atomically create `key` bound to `lease_id`, only if it does not@@ -371,7 +429,7 @@ impl EtcdStore {value,Some(PutOptions::new().with_lease(lease_id)),)]);- let resp = self.client.clone().txn(txn).await?;+ let resp = self.observe(self.client.clone().txn(txn).await)?;Ok(resp.succeeded())}@@ -379,7 +437,7 @@ impl EtcdStore {pub async fn grant_lease(&self, ttl: i64) -> Result<i64> {let _t = OpTimer::new("grant_lease");- let resp = self.client.clone().lease_grant(ttl, None).await?;+ let resp = self.observe(self.client.clone().lease_grant(ttl, None).await)?;Ok(resp.id())}@@ -388,13 +446,14 @@ impl EtcdStore {lease_id: i64,) -> Result<(etcd_client::LeaseKeeper, etcd_client::LeaseKeepAliveStream)> {let _t = OpTimer::new("keep_alive");- let (keeper, stream) = self.client.clone().lease_keep_alive(lease_id).await?;+ let (keeper, stream) =+ self.observe(self.client.clone().lease_keep_alive(lease_id).await)?;Ok((keeper, stream))}pub async fn revoke_lease(&self, lease_id: i64) -> Result<()> {let _t = OpTimer::new("revoke_lease");- self.client.clone().lease_revoke(lease_id).await?;+ self.observe(self.client.clone().lease_revoke(lease_id).await)?;Ok(())}@@ -419,3 +478,106 @@ pub fn parse_watch_value<T: DeserializeOwned>(.ok_or_else(|| Error::InvalidState("watch event missing kv".to_string()))?;serde_json::from_slice(kv.value()).map_err(Error::from)}++#[cfg(test)]+mod member_failover_tests {+ use std::time::Instant;++ use super::*;++ const LIVE_ETCD: &str = "http://localhost:2379";+ const DEAD_ENDPOINT: &str = "http://127.0.0.1:1";++ async fn store_on_dead_member(known_members: Vec<String>) -> EtcdStore {+ let nanos = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .unwrap()+ .as_nanos();+ let store = EtcdStore::connect_with_member_failover(StoreConfig {+ endpoints: vec![DEAD_ENDPOINT.to_string()],+ prefix: format!("/test-member-failover-{nanos}/"),+ })+ .await+ .expect("connect is lazy, so a dead endpoint still yields a store");+ store+ .failover+ .as_ref()+ .unwrap()+ .set_members_for_test(known_members);+ store+ }++ #[tokio::test]+ async fn a_store_on_a_failing_member_moves_to_a_healthy_one() {+ let store = store_on_dead_member(vec![LIVE_ETCD.to_string()]).await;++ let deadline = Instant::now() + Duration::from_secs(15);+ loop {+ if store.get_raw("/probe").await.is_ok() {+ break;+ }+ assert!(+ Instant::now() < deadline,+ "store never left the dead member"+ );+ tokio::time::sleep(Duration::from_millis(100)).await;+ }+ assert_eq!(+ store.failover.as_ref().unwrap().current_for_test(),+ vec![LIVE_ETCD.to_string()]+ );+ }++ #[tokio::test]+ async fn a_store_without_a_member_list_reads_one_after_a_call_succeeds() {+ let store = EtcdStore::connect_with_member_failover(StoreConfig {+ endpoints: vec![LIVE_ETCD.to_string()],+ prefix: "/test-member-failover-refresh/".to_string(),+ })+ .await+ .unwrap();+ let failover = store.failover.as_ref().unwrap();+ failover.set_members_for_test(Vec::new());++ store.get_raw("/probe").await.unwrap();++ let deadline = Instant::now() + Duration::from_secs(5);+ while failover.members_for_test().is_empty() {+ assert!(Instant::now() < deadline, "member list never refreshed");+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ }++ #[tokio::test]+ async fn only_a_long_unanswered_call_switches_at_once() {+ let store = store_on_dead_member(vec![LIVE_ETCD.to_string()]).await;+ let failover = store.failover.as_ref().unwrap();++ store.observe_member_unanswered(Duration::from_secs(1));+ assert!(!failover.switching_for_test());++ store.observe_member_unanswered(Duration::from_secs(2));+ let deadline = Instant::now() + Duration::from_secs(10);+ while failover.current_for_test() != vec![LIVE_ETCD.to_string()] {+ assert!(+ Instant::now() < deadline,+ "store never left the dead member"+ );+ tokio::time::sleep(Duration::from_millis(50)).await;+ }+ }++ #[tokio::test]+ async fn a_store_stays_put_when_no_other_member_answers() {+ let store = store_on_dead_member(vec!["http://127.0.0.1:2".to_string()]).await;++ for _ in 0..6 {+ assert!(store.get_raw("/probe").await.is_err());+ }+ tokio::time::sleep(Duration::from_secs(4)).await;+ assert_eq!(+ store.failover.as_ref().unwrap().current_for_test(),+ vec![DEAD_ENDPOINT.to_string()]+ );+ }+}diff --git a/rust/personhog-changelog-relay/src/main.rs b/rust/personhog-changelog-relay/src/main.rsindex 5eec6709a903..8c543abdab71 100644--- a/rust/personhog-changelog-relay/src/main.rs+++ b/rust/personhog-changelog-relay/src/main.rs@@ -94,7 +94,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {.expect("Metrics server error");});- let store = EtcdStore::connect(StoreConfig {+ let store = EtcdStore::connect_with_member_failover(StoreConfig {endpoints: config.etcd_endpoint_list(),prefix: config.etcd_prefix.clone(),})diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.mdindex f5b2eedfa791..1f7dbb57d2ac 100644--- a/rust/personhog-coordination/README.md+++ b/rust/personhog-coordination/README.md@@ -94,7 +94,8 @@ Several failure modes are handled explicitly:- **Warm provenance**: A pod records what each partition warm was installed for (serving, or a specific handoff id). Warming honors only a warm installed for the *current* handoff — one left over from an earlier ownership era is released and rebuilt, because its cache can hold values that predate writes the changelog accepted since, and a fresh warm replays only from the writer's committed offset. Without this, a returning owner whose `Released` convergence never ran could ack a warm it never performed and lose acknowledged writes — a counterexample the stateright model found and now pins.- **Coordinator restart**: On startup, the coordinator runs `reconcile_pending_handoffs`. For every existing handoff it applies cleanup if `Complete`, otherwise calls `check_phase_advance` to nudge the state machine. This handles the case where acks were written before the coordinator was up — the ack-watch only fires on Put events, so without reconcile those acks would be invisible.- **Reconcile tick**: Beyond the startup reconcile, the coordinator re-evaluates every in-flight handoff on a fixed interval (`reconcile_interval`, default 5 s). This is the liveness backstop for state changes that fire no watched event — e.g. a router departing (nothing watches router registrations) newly satisfying a freeze quorum. Everything the tick drives is idempotent: phase transitions use CAS and cleanup tolerates already-deleted records.-- **Lease loss (self-fencing)**: Connection trouble and lease loss are different things. A broken or silent keepalive stream is a fact about one connection — the lease lives in etcd's replicated keyspace and stays valid until its TTL passes without renewal — so ambiguous keepalive failures rebuild the stream and retry (a fresh connection through the service reaches a healthy member) for as long as the last confirmed renewal is recent; a routine single-member restart costs some retries (`personhog_coordination_keepalive_retries_total{component}`), not a fence. The keepalive declares loss on exactly two signals: etcd answering a round with TTL ≤ 0 (authoritative — the lease is revoked or expired), or the renewal margin running out (two thirds of the TTL without a confirmed renewal — the final third is reserved so the fence completes before the coordinator can possibly treat the lease as expired; retrying past it would gamble split-brain against availability). On loss the component ends the coordination attempt immediately, skipping the graceful drain: the coordinator already considers it dead and is reassigning its partitions via the dead-owner path, so a graceful drain would only race that. A pod self-fences *locally* before anything else: every held partition is drained and, once quiesced, released (dropping its cache and serving authority — re-acquisition always re-warms). A partition whose drain cannot quiesce is deliberately **not** released — release is what unfences admissions and drops the cache, and a leaseless pod must do neither over live in-flight work — so it stays fenced and held, in-place recovery is refused, and the process restart clears it by death. A zombie owner therefore never keeps serving partitions the protocol has reassigned beyond the window that restart closes. Lease loss is observed at every await the session makes while holding a lease — the attempt select, the backoff nap between failed attempts, and the graceful drain all race the keepalive — so once the keepalive declares loss, the fence is never deferred by supervisor state. Every other session end fences too — budget exhaustion and shutdown revoke the lease, which deletes the registration instantly, so the local fence runs first (a no-op after a completed graceful drain) and the dead-owner path never meets a still-serving owner. On lease loss the fence's drain budget is the runway the keepalive margin reserved (a third of the TTL), not the full graceful drain timeout; overshoot poisons recovery and the process restart clears stragglers by death.+- **Lease loss (self-fencing)**: Connection trouble and lease loss are different things. A broken or silent keepalive stream is a fact about one connection — the lease lives in etcd's replicated keyspace and stays valid until its TTL passes without renewal — so ambiguous keepalive failures rebuild the stream and retry (the store moves to another member when its own keeps failing; see `assignment_coordination::failover`) for as long as the last confirmed renewal is recent; a routine single-member restart costs some retries (`personhog_coordination_keepalive_retries_total{component}`), not a fence. The keepalive declares loss on exactly two signals: etcd answering a round with TTL ≤ 0 (authoritative — the lease is revoked or expired), or the renewal margin running out (two thirds of the TTL without a confirmed renewal — the final third is reserved so the fence completes before the coordinator can possibly treat the lease as expired; retrying past it would gamble split-brain against availability). On loss the component ends the coordination attempt immediately, skipping the graceful drain: the coordinator already considers it dead and is reassigning its partitions via the dead-owner path, so a graceful drain would only race that. A pod self-fences *locally* before anything else: every held partition is drained and, once quiesced, released (dropping its cache and serving authority — re-acquisition always re-warms). A partition whose drain cannot quiesce is deliberately **not** released — release is what unfences admissions and drops the cache, and a leaseless pod must do neither over live in-flight work — so it stays fenced and held, in-place recovery is refused, and the process restart clears it by death. A zombie owner therefore never keeps serving partitions the protocol has reassigned beyond the window that restart closes. Lease loss is observed at every await the session makes while holding a lease — the attempt select, the backoff nap between failed attempts, and the graceful drain all race the keepalive — so once the keepalive declares loss, the fence is never deferred by supervisor state. Every other session end fences too — budget exhaustion and shutdown revoke the lease, which deletes the registration instantly, so the local fence runs first (a no-op after a completed graceful drain) and the dead-owner path never meets a still-serving owner. On lease loss the fence's drain budget is the runway the keepalive margin reserved (a third of the TTL), not the full graceful drain timeout; overshoot poisons recovery and the process restart clears stragglers by death.+- **etcd member failover**: Each process keeps one connection to whichever etcd member it reached first. Three member-level failures in a row (Unavailable, DeadlineExceeded, a broken connection), or one call left unanswered for at least 2 s, start a switch: the store probes the other members from its cached member list with a linearizable read and moves new calls to the first that answers. Requests carry etcd's require-leader flag, so a member without a leader rejects them at once. Watches and keepalive streams already open stay on their connection until it closes or etcd cancels them. A renewal waits up to the heartbeat interval for an answer, so a member that stops answering without closing its connections still costs that pod one renewal margin, and with it the pod's partitions.- **Supervised coordination runs (etcd resilience)**: A failed coordination attempt — a broken watch stream, a failed etcd write, an exhausted reconcile budget — no longer kills the process; the run supervisors tear the attempt down cleanly, back off exponentially, and rebuild through the same bootstrap that recovers a restarted process, while the data plane keeps serving. The two participant classes recover *differently*, on purpose. A **router** sheds its registration on every failed attempt (its registration is a freeze-quorum vote, and a registered but non-acking router would stall every quorum until the phase deadline); serving through its gap is safe because handoffs created while it is unregistered exclude it from their quorums and the leader-side fences reject its stale forwards. A **pod** does the opposite: its lease session (grant, register, heartbeat) lives *above* the attempt loop and survives attempt failures, so the coordinator keeps seeing a live pod and never runs the dead-owner path against a process that is still serving — the invariant is that serving authority is held exactly as long as the lease-backed registration behind it. This matters because the dead-owner path has no fence: its safety rests entirely on a deregistered owner being unable to produce, so a pod must never voluntarily deregister while it still serves. Only lease loss ends a pod session, self-fencing locally (every held partition released; re-acquisition re-warms) before a fresh session registers anew. A consecutive-failure budget (`ROUTER_RUN_RETRY_BUDGET`, default 10; same default on pods) keeps a genuinely wedged participant on the process-restart path; an attempt that made real progress — a convergence applied, a reconcile pass completed — resets the count, and the reset is keyed on measured progress rather than elapsed time because a time threshold silently exempts every failure detector slower than it (the reconcile budget and the stall watchdog both take a minute to fire), and `personhog_coordination_run_restarts_total{component}` counts every rebuild.### Concurrency and atomicitydiff --git a/rust/personhog-coordination/src/error.rs b/rust/personhog-coordination/src/error.rsindex 6c1198cd838a..1685c896016d 100644--- a/rust/personhog-coordination/src/error.rs+++ b/rust/personhog-coordination/src/error.rs@@ -35,6 +35,14 @@ impl Error {Self::Store(assignment_coordination::error::Error::LeadershipLost)}+ /// The etcd error underneath, when etcd is what failed.+ pub fn etcd_error(&self) -> Option<&etcd_client::Error> {+ match self {+ Self::Store(assignment_coordination::error::Error::Etcd(e)) => Some(e),+ _ => None,+ }+ }+/// Whether losing the election lease is what ended this term. An/// abdication is the protocol working — a successor takes over and/// reconciles — so it is not evidence that this process is unhealthy.diff --git a/rust/personhog-coordination/src/util.rs b/rust/personhog-coordination/src/util.rsindex 1d8f69377edd..545556080936 100644--- a/rust/personhog-coordination/src/util.rs+++ b/rust/personhog-coordination/src/util.rs@@ -148,15 +148,19 @@ pub async fn run_lease_keepalive(if left.is_zero() {return Err(margin_exhausted());}- let attempt = tokio::time::timeout(left.min(interval), store.keep_alive(lease_id));+ let wait = left.min(interval);+ let attempt = tokio::time::timeout(wait, store.keep_alive(lease_id));let failure = tokio::select! {_ = cancel.cancelled() => return Ok(()),r = attempt => match r {Ok(Ok(pair)) => break pair,Ok(Err(e)) => e,- Err(_) => Error::invalid_state(- "keepalive stream establishment timed out".to_string(),- ),+ Err(_) => {+ store.inner().observe_member_unanswered(wait);+ Error::invalid_state(+ "keepalive stream establishment timed out".to_string(),+ )+ }},};counter!(@@ -219,12 +223,24 @@ pub async fn run_lease_keepalive(// catches those.crate::store::count_call("keep_alive_renewal");let _renewal = assignment_coordination::store::OpTimer::new("keep_alive_renewal");- match tokio::time::timeout(left.min(interval), round).await {- Ok(r) => r,- Err(_) => Err(Error::invalid_state(format!(- "keepalive round unanswered within {:?}",- left.min(interval)- ))),+ let wait = left.min(interval);+ match tokio::time::timeout(wait, round).await {+ Ok(r) => {+ match &r {+ Ok(_) => store.inner().observe_member_outcome(Ok(())),+ Err(e) => match e.etcd_error() {+ Some(etcd) => store.inner().observe_member_outcome(Err(etcd)),+ None => store.inner().observe_member_failure(),+ },+ }+ r+ }+ Err(_) => {+ store.inner().observe_member_unanswered(wait);+ Err(Error::invalid_state(format!(+ "keepalive round unanswered within {wait:?}"+ )))+ }}};match outcome {diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rsindex 87ffde0cada4..8446092f723c 100644--- a/rust/personhog-identity/src/main.rs+++ b/rust/personhog-identity/src/main.rs@@ -148,7 +148,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {// The delete saga groups its fence calls by the leaders' partition// count, which lives in etcd; a service that cannot read it would// batch wrong, so refuse to start instead.- let etcd_store = EtcdStore::connect(StoreConfig {+ let etcd_store = EtcdStore::connect_with_member_failover(StoreConfig {endpoints: config.etcd_endpoint_list(),prefix: config.etcd_prefix.clone(),})diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rsindex ba06cb9c5e3b..db0c09d6bfbb 100644--- a/rust/personhog-leader/src/main.rs+++ b/rust/personhog-leader/src/main.rs@@ -361,7 +361,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {endpoints: config.etcd_endpoint_list(),prefix: config.etcd_prefix.clone(),};- let etcd_store = EtcdStore::connect(etcd_config)+ let etcd_store = EtcdStore::connect_with_member_failover(etcd_config).await.expect("Failed to connect to etcd");let store = Arc::new(PersonhogStore::new(etcd_store));diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rsindex c19739c3f5ab..dfd599040890 100644--- a/rust/personhog-router/src/main.rs+++ b/rust/personhog-router/src/main.rs@@ -253,7 +253,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {endpoints: config.etcd_endpoint_list(),prefix: config.etcd_prefix.clone(),};- let etcd_store = EtcdStore::connect(etcd_config)+ let etcd_store = EtcdStore::connect_with_member_failover(etcd_config).await.expect("Failed to connect to etcd");let store = Arc::new(PersonhogStore::new(etcd_store));diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rsindex 0e10d22c0618..659541589f8b 100644--- a/rust/personhog-writer/src/main.rs+++ b/rust/personhog-writer/src/main.rs@@ -55,7 +55,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {tracing::info!("Upsert concurrency: {}", config.upsert_concurrency);tracing::info!("Metrics port: {}", config.metrics_port);- let store = EtcdStore::connect(StoreConfig {+ let store = EtcdStore::connect_with_member_failover(StoreConfig {endpoints: config.etcd_endpoint_list(),prefix: config.etcd_prefix.clone(),})