Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): switch etcd members when the current one cannot serve (#123639)

Zach Brown8e699ae37aa5
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 52c49e18bf24..b770728b17ec 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -552,12 +552,14 @@ name = "assignment-coordination"
version = "0.1.0"
dependencies = [
"etcd-client",
+ "h2 0.4.13",
"metrics",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"tokio-util",
+ "tonic 0.14.6",
"tracing",
]
diff --git a/rust/common/assignment-coordination/Cargo.toml b/rust/common/assignment-coordination/Cargo.toml
index e3a5aa8e0930..f43e05ea3a7c 100644
--- a/rust/common/assignment-coordination/Cargo.toml
+++ b/rust/common/assignment-coordination/Cargo.toml
@@ -6,12 +6,14 @@ publish = false
[dependencies]
etcd-client = "0.18"
+h2 = "0.4"
metrics = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio-util = { version = "0.7", features = ["rt"] }
+tonic = { version = "0.14", default-features = false, features = ["transport"] }
tracing = { workspace = true }
[lints]
diff --git a/rust/common/assignment-coordination/src/failover.rs b/rust/common/assignment-coordination/src/failover.rs
new file mode 100644
index 000000000000..13c1a050b16c
--- /dev/null
+++ b/rust/common/assignment-coordination/src/failover.rs
@@ -0,0 +1,374 @@
+use std::collections::hash_map::RandomState;
+use std::hash::{BuildHasher, Hasher};
+use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use etcd_client::{Client, ConnectOptions, GetOptions};
+use tonic::Code;
+
+/// Consecutive member-level failures that start a switch attempt.
+const FAILURE_THRESHOLD: u32 = 3;
+/// Minimum gap between switch attempts, so a cluster-wide outage does not
+/// turn every failed call into a round of probes.
+const MIN_ATTEMPT_INTERVAL: Duration = Duration::from_secs(5);
+/// Budget for one probe: connect, one linearizable read, one member list.
+const PROBE_TIMEOUT: Duration = Duration::from_secs(3);
+/// An unanswered call that waited at least this long starts a switch at once.
+/// Shorter waits, such as the coordinator's 1s renewals on a busy cluster,
+/// count toward FAILURE_THRESHOLD instead.
+const UNANSWERED_FLOOR: Duration = Duration::from_secs(2);
+
+/// Whether an etcd error says this member cannot serve: Unavailable,
+/// DeadlineExceeded, or a transport failure under the call. Errors about the
+/// request itself (NOSPACE, a failed compare, a compacted revision) do not.
+pub fn is_member_level(err: &etcd_client::Error) -> bool {
+ match err {
+ etcd_client::Error::GRpcStatus(status) => {
+ matches!(status.code(), Code::Unavailable | Code::DeadlineExceeded)
+ || raised_by_connection(status)
+ }
+ etcd_client::Error::TransportError(_) | etcd_client::Error::IoError(_) => true,
+ _ => false,
+ }
+}
+
+/// Whether the client raised this status for a failed connection or HTTP/2
+/// stream, whatever its code, rather than etcd sending it.
+fn raised_by_connection(status: &tonic::Status) -> bool {
+ let mut source = std::error::Error::source(status);
+ while let Some(err) = source {
+ if err.is::<tonic::transport::Error>()
+ || err.is::<h2::Error>()
+ || err.is::<std::io::Error>()
+ {
+ return true;
+ }
+ source = err.source();
+ }
+ false
+}
+
+struct State {
+ /// Endpoints currently in the client's balanced channel.
+ current: Vec<String>,
+ /// The endpoints the store was configured with, used when no member
+ /// list is known.
+ configured: Vec<String>,
+ /// One client URL per voting member, from the last member list read.
+ members: Vec<String>,
+ last_attempt: Option<Instant>,
+ last_failed_refresh: Option<Instant>,
+}
+
+/// Clears a flag when dropped, so a task that panics cannot leave it set.
+struct ClearOnDrop(Arc<AtomicBool>);
+
+impl Drop for ClearOnDrop {
+ fn drop(&mut self) {
+ self.0.store(false, Ordering::Release);
+ }
+}
+
+/// Moves a client to a different etcd member when the one it is connected
+/// to keeps failing while the rest of the cluster may still be healthy.
+pub(crate) struct MemberFailover {
+ client: Client,
+ probe_options: ConnectOptions,
+ consecutive_failures: AtomicU32,
+ switching: Arc<AtomicBool>,
+ refreshing: Arc<AtomicBool>,
+ members_known: AtomicBool,
+ state: Mutex<State>,
+}
+
+impl MemberFailover {
+ pub(crate) fn new(client: Client, options: ConnectOptions, endpoints: Vec<String>) -> Self {
+ Self {
+ client,
+ probe_options: options
+ .with_require_leader(true)
+ .with_connect_timeout(PROBE_TIMEOUT),
+ consecutive_failures: AtomicU32::new(0),
+ switching: Arc::new(AtomicBool::new(false)),
+ refreshing: Arc::new(AtomicBool::new(false)),
+ members_known: AtomicBool::new(false),
+ state: Mutex::new(State {
+ current: endpoints.clone(),
+ configured: endpoints,
+ members: Vec::new(),
+ last_attempt: None,
+ last_failed_refresh: None,
+ }),
+ }
+ }
+
+ /// Reads the member list through the connected client, best effort.
+ pub(crate) async fn refresh_members(&self) {
+ let mut client = self.client.clone();
+ let members = match tokio::time::timeout(PROBE_TIMEOUT, client.member_list()).await {
+ Ok(Ok(resp)) => member_urls(&resp),
+ _ => Vec::new(),
+ };
+ if members.is_empty() {
+ self.lock().last_failed_refresh = Some(Instant::now());
+ } else {
+ self.set_members(members);
+ }
+ }
+
+ pub(crate) fn note_success(self: &Arc<Self>) {
+ self.consecutive_failures.store(0, Ordering::Relaxed);
+ if !self.members_known.load(Ordering::Acquire) {
+ self.start_refresh();
+ }
+ }
+
+ /// Reads the member list in the background once a call succeeds, for a
+ /// store that started while etcd could not answer.
+ fn start_refresh(self: &Arc<Self>) {
+ if self.refreshing.swap(true, Ordering::AcqRel) {
+ return;
+ }
+ let latch = ClearOnDrop(Arc::clone(&self.refreshing));
+ if self
+ .lock()
+ .last_failed_refresh
+ .is_some_and(|at| at.elapsed() < MIN_ATTEMPT_INTERVAL)
+ {
+ return;
+ }
+ let this = Arc::clone(self);
+ tokio::spawn(async move {
+ let _latch = latch;
+ this.refresh_members().await;
+ });
+ }
+
+ pub(crate) fn note_failure(self: &Arc<Self>) {
+ let failures = self.consecutive_failures.fetch_add(1, Ordering::Relaxed) + 1;
+ if failures >= FAILURE_THRESHOLD {
+ self.start_switch();
+ }
+ }
+
+ /// A call that got no answer after `waited`.
+ pub(crate) fn note_unanswered(self: &Arc<Self>, waited: Duration) {
+ if waited < UNANSWERED_FLOOR {
+ self.note_failure();
+ return;
+ }
+ self.consecutive_failures.fetch_add(1, Ordering::Relaxed);
+ self.start_switch();
+ }
+
+ fn start_switch(self: &Arc<Self>) {
+ if self.switching.swap(true, Ordering::AcqRel) {
+ return;
+ }
+ let latch = ClearOnDrop(Arc::clone(&self.switching));
+ let this = Arc::clone(self);
+ tokio::spawn(async move {
+ let _latch = latch;
+ this.attempt_switch().await;
+ });
+ }
+
+ async fn attempt_switch(&self) {
+ let (current, candidates) = {
+ let mut state = self.lock();
+ if state
+ .last_attempt
+ .is_some_and(|at| at.elapsed() < MIN_ATTEMPT_INTERVAL)
+ {
+ return;
+ }
+ state.last_attempt = Some(Instant::now());
+ let candidates: Vec<String> = state
+ .members
+ .iter()
+ .filter(|url| !state.current.contains(url))
+ .cloned()
+ .collect();
+ (state.current.clone(), candidates)
+ };
+
+ if candidates.is_empty() {
+ // No other member is known: reconnect so the configured names
+ // resolve again.
+ let configured = self.lock().configured.clone();
+ tracing::warn!(
+ endpoints = ?configured,
+ "etcd member keeps failing and no other member is known; reconnecting"
+ );
+ let outcome = match self.replace_endpoints(&current, &configured).await {
+ Ok(()) => "reconnected",
+ Err(()) => "swap_failed",
+ };
+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => outcome)
+ .increment(1);
+ return;
+ }
+
+ for candidate in rotated(candidates) {
+ match self.probe(&candidate).await {
+ Ok(members) => {
+ if self
+ .replace_endpoints(&current, std::slice::from_ref(&candidate))
+ .await
+ .is_err()
+ {
+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "swap_failed")
+ .increment(1);
+ return;
+ }
+ if !members.is_empty() {
+ self.set_members(members);
+ }
+ self.consecutive_failures.store(0, Ordering::Relaxed);
+ tracing::warn!(
+ from = ?current,
+ to = %candidate,
+ "switched etcd member after repeated failures"
+ );
+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "switched")
+ .increment(1);
+ return;
+ }
+ Err(error) => {
+ tracing::warn!(%candidate, %error, "etcd member probe failed");
+ }
+ }
+ }
+ tracing::warn!(
+ current = ?current,
+ "etcd member keeps failing and no other member answered a probe; staying"
+ );
+ metrics::counter!("assignment_coordination_etcd_member_switches_total", "outcome" => "no_healthy_member")
+ .increment(1);
+ }
+
+ /// Connects to `url` alone and checks it can serve a linearizable read,
+ /// which needs a leader and a quorum. Returns its view of the members.
+ async fn probe(&self, url: &str) -> Result<Vec<String>, String> {
+ let attempt = async {
+ let mut client = Client::connect([url], Some(self.probe_options.clone())).await?;
+ client
+ .get("probe", Some(GetOptions::new().with_count_only()))
+ .await?;
+ let members = client.member_list().await?;
+ Ok::<_, etcd_client::Error>(member_urls(&members))
+ };
+ match tokio::time::timeout(PROBE_TIMEOUT, attempt).await {
+ Ok(Ok(members)) => Ok(members),
+ Ok(Err(error)) => Err(error.to_string()),
+ Err(_) => Err(format!("no answer within {PROBE_TIMEOUT:?}")),
+ }
+ }
+
+ /// Inserts the new endpoints before removing the old ones, so the
+ /// balanced channel is never empty. Fails, leaving the old endpoints in
+ /// place, when an insert fails.
+ async fn replace_endpoints(&self, old: &[String], new: &[String]) -> Result<(), ()> {
+ for url in new {
+ if let Err(error) = self.client.add_endpoint(url).await {
+ tracing::error!(%url, %error, "failed to add etcd endpoint; keeping the current one");
+ return Err(());
+ }
+ }
+ let mut current = new.to_vec();
+ for url in old.iter().filter(|url| !new.contains(url)) {
+ if let Err(error) = self.client.remove_endpoint(url).await {
+ tracing::error!(%url, %error, "failed to remove etcd endpoint");
+ current.push(url.clone());
+ }
+ }
+ self.lock().current = current;
+ Ok(())
+ }
+
+ fn set_members(&self, members: Vec<String>) {
+ self.members_known
+ .store(!members.is_empty(), Ordering::Release);
+ self.lock().members = members;
+ }
+
+ fn lock(&self) -> std::sync::MutexGuard<'_, State> {
+ self.state
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ }
+
+ #[cfg(test)]
+ pub(crate) fn set_members_for_test(&self, members: Vec<String>) {
+ self.set_members(members);
+ }
+
+ #[cfg(test)]
+ pub(crate) fn current_for_test(&self) -> Vec<String> {
+ self.lock().current.clone()
+ }
+
+ #[cfg(test)]
+ pub(crate) fn members_for_test(&self) -> Vec<String> {
+ self.lock().members.clone()
+ }
+
+ #[cfg(test)]
+ pub(crate) fn switching_for_test(&self) -> bool {
+ self.switching.load(Ordering::Acquire)
+ }
+}
+
+fn member_urls(resp: &etcd_client::MemberListResponse) -> Vec<String> {
+ resp.members()
+ .iter()
+ .filter(|member| !member.is_learner())
+ .filter_map(|member| member.client_urls().first().cloned())
+ .collect()
+}
+
+/// Starts the candidate list at a random member so processes leaving the
+/// same member spread across the others.
+fn rotated(mut candidates: Vec<String>) -> Vec<String> {
+ let mut hasher = RandomState::new().build_hasher();
+ hasher.write_u64(0);
+ let start = (hasher.finish() as usize) % candidates.len();
+ candidates.rotate_left(start);
+ candidates
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn status(code: Code) -> etcd_client::Error {
+ etcd_client::Error::GRpcStatus(tonic::Status::new(code, "test"))
+ }
+
+ #[test]
+ fn member_level_errors_are_the_ones_a_different_member_could_avoid() {
+ assert!(is_member_level(&status(Code::Unavailable)));
+ assert!(is_member_level(&status(Code::DeadlineExceeded)));
+ assert!(is_member_level(&etcd_client::Error::IoError(
+ std::io::Error::from(std::io::ErrorKind::ConnectionReset)
+ )));
+ let broken_stream =
+ tonic::Status::from_error(Box::new(h2::Error::from(h2::Reason::STREAM_CLOSED)));
+ assert_eq!(broken_stream.code(), Code::Unknown);
+ assert!(is_member_level(&etcd_client::Error::GRpcStatus(
+ broken_stream
+ )));
+ for code in [
+ Code::Unknown,
+ Code::ResourceExhausted,
+ Code::FailedPrecondition,
+ Code::NotFound,
+ Code::InvalidArgument,
+ Code::OutOfRange,
+ Code::PermissionDenied,
+ ] {
+ assert!(!is_member_level(&status(code)), "{code:?}");
+ }
+ }
+}
diff --git a/rust/common/assignment-coordination/src/lib.rs b/rust/common/assignment-coordination/src/lib.rs
index 0dedaa1b60a9..8f276262df95 100644
--- a/rust/common/assignment-coordination/src/lib.rs
+++ b/rust/common/assignment-coordination/src/lib.rs
@@ -1,4 +1,5 @@
pub mod error;
+pub mod failover;
pub mod leader_election;
pub mod store;
pub mod strategy;
diff --git a/rust/common/assignment-coordination/src/store.rs b/rust/common/assignment-coordination/src/store.rs
index 569be9056096..87a110fd18cf 100644
--- a/rust/common/assignment-coordination/src/store.rs
+++ b/rust/common/assignment-coordination/src/store.rs
@@ -1,3 +1,4 @@
+use std::sync::Arc;
use std::time::Duration;
use etcd_client::{
@@ -8,6 +9,7 @@ use serde::de::DeserializeOwned;
use serde::Serialize;
use crate::error::{Error, Result};
+use crate::failover::{is_member_level, MemberFailover};
#[derive(Debug, Clone)]
pub struct StoreConfig {
@@ -27,6 +29,7 @@ pub struct StoreConfig {
pub struct EtcdStore {
client: Client,
config: StoreConfig,
+ failover: Option<Arc<MemberFailover>>,
}
/// Records one etcd operation's wall time on drop, so every return path
@@ -68,39 +71,94 @@ fn record_payload_bytes(op: &'static str, bytes: usize) {
.record(bytes as f64);
}
+// HTTP/2 pings fail a silent connection in about 15s. A per-request timeout
+// would also end healthy watch streams, so there is none. The interval stays
+// above etcd's 5s keepalive floor, and idle pings stay off because etcd
+// counts them as strikes.
+fn connect_options() -> ConnectOptions {
+ ConnectOptions::new()
+ .with_connect_timeout(Duration::from_secs(5))
+ .with_keep_alive(Duration::from_secs(10), Duration::from_secs(5))
+ .with_keep_alive_while_idle(false)
+}
+
fn kvs_bytes(kvs: &[etcd_client::KeyValue]) -> usize {
kvs.iter().map(|kv| kv.key().len() + kv.value().len()).sum()
}
impl EtcdStore {
pub async fn connect(config: StoreConfig) -> Result<Self> {
- // Transport-level liveness so a silent network partition fails
- // fast instead of hanging until TCP retransmission gives up
- // (minutes — far past any lease TTL): HTTP/2 pings ride every
- // connection, including idle ones and long-lived watch streams,
- // and error all in-flight requests within roughly one ping
- // interval plus its timeout of the peer going dark. Deliberately
- // no per-request timeout — it would apply to the whole lifetime
- // of a watch stream and kill healthy watches.
- // The ping interval must clear etcd's server-side gRPC keepalive
- // enforcement (--grpc-keepalive-min-time, default 5s): pings at or
- // under the floor are strikes, and two strikes close the
- // connection with GOAWAY. Idle pings are likewise strikes unless
- // the server permits them, so they stay off — every component
- // that matters holds an active watch or keepalive stream, and an
- // idle channel is revalidated on next use within the connect
- // timeout.
- let options = ConnectOptions::new()
- .with_connect_timeout(Duration::from_secs(5))
- .with_keep_alive(Duration::from_secs(10), Duration::from_secs(5))
- .with_keep_alive_while_idle(false);
- let client = Client::connect(&config.endpoints, Some(options)).await?;
- Ok(Self { client, config })
+ let client = Client::connect(&config.endpoints, Some(connect_options())).await?;
+ Ok(Self {
+ client,
+ config,
+ failover: None,
+ })
+ }
+
+ /// Like `connect`, and moves to another member when this one keeps
+ /// failing. Requests carry etcd's require-leader flag, so a member
+ /// without a leader rejects them at once.
+ pub async fn connect_with_member_failover(config: StoreConfig) -> Result<Self> {
+ let options = connect_options().with_require_leader(true);
+ let client = Client::connect(&config.endpoints, Some(options.clone())).await?;
+ let failover = Arc::new(MemberFailover::new(
+ client.clone(),
+ options,
+ config.endpoints.clone(),
+ ));
+ failover.refresh_members().await;
+ Ok(Self {
+ client,
+ config,
+ failover: Some(failover),
+ })
}
/// Shares an existing client's channel instead of opening another.
pub fn from_client(client: Client, config: StoreConfig) -> Self {
- Self { client, config }
+ Self {
+ client,
+ config,
+ failover: None,
+ }
+ }
+
+ /// Records a call made outside this store's methods, such as a lease
+ /// renewal on a `LeaseKeeper`, so it counts toward member failover.
+ pub fn observe_member_outcome(&self, outcome: std::result::Result<(), &etcd_client::Error>) {
+ let Some(failover) = &self.failover else {
+ return;
+ };
+ match outcome {
+ Ok(()) => failover.note_success(),
+ Err(err) if is_member_level(err) => failover.note_failure(),
+ Err(_) => {}
+ }
+ }
+
+ /// Records a call that got no answer within `waited`, such as a lease
+ /// renewal. A long enough wait starts a switch attempt at once.
+ pub fn observe_member_unanswered(&self, waited: Duration) {
+ if let Some(failover) = &self.failover {
+ failover.note_unanswered(waited);
+ }
+ }
+
+ /// Records a call outside this store that failed without an etcd error,
+ /// such as a keepalive stream that closed.
+ pub fn observe_member_failure(&self) {
+ if let Some(failover) = &self.failover {
+ failover.note_failure();
+ }
+ }
+
+ fn observe<T>(
+ &self,
+ result: std::result::Result<T, etcd_client::Error>,
+ ) -> std::result::Result<T, etcd_client::Error> {
+ self.observe_member_outcome(result.as_ref().map(|_| ()));
+ result
}
pub fn prefix(&self) -> &str {
@@ -115,7 +173,7 @@ impl EtcdStore {
pub async fn get_raw(&self, key: &str) -> Result<Option<Vec<u8>>> {
let _t = OpTimer::new("get_raw");
- let resp = self.client.clone().get(key, None).await?;
+ let resp = self.observe(self.client.clone().get(key, None).await)?;
record_payload_bytes("get_raw", kvs_bytes(resp.kvs()));
Ok(resp.kvs().first().map(|kv| kv.value().to_vec()))
}
@@ -124,7 +182,7 @@ impl EtcdStore {
let _t = OpTimer::new("put_raw");
let value = value.into();
record_payload_bytes("put_raw", key.len() + value.len());
- self.client.clone().put(key, value, None).await?;
+ self.observe(self.client.clone().put(key, value, None).await)?;
Ok(())
}
@@ -132,7 +190,7 @@ impl EtcdStore {
pub async fn get<T: DeserializeOwned>(&self, key: &str) -> Result<Option<T>> {
let _t = OpTimer::new("get");
- let resp = self.client.clone().get(key, None).await?;
+ let resp = self.observe(self.client.clone().get(key, None).await)?;
record_payload_bytes("get", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => Ok(Some(serde_json::from_slice(kv.value())?)),
@@ -142,7 +200,7 @@ impl EtcdStore {
pub async fn get_versioned<T: DeserializeOwned>(&self, key: &str) -> Result<Option<(T, i64)>> {
let _t = OpTimer::new("get_versioned");
- let resp = self.client.clone().get(key, None).await?;
+ let resp = self.observe(self.client.clone().get(key, None).await)?;
record_payload_bytes("get_versioned", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => {
@@ -164,7 +222,7 @@ impl EtcdStore {
key: &str,
) -> Result<Option<(T, i64)>> {
let _t = OpTimer::new("get_with_mod_revision");
- let resp = self.client.clone().get(key, None).await?;
+ let resp = self.observe(self.client.clone().get(key, None).await)?;
record_payload_bytes("get_with_mod_revision", kvs_bytes(resp.kvs()));
match resp.kvs().first() {
Some(kv) => {
@@ -185,7 +243,7 @@ impl EtcdStore {
key: &str,
) -> Result<(Option<T>, i64)> {
let _t = OpTimer::new("get_with_revision");
- let resp = self.client.clone().get(key, None).await?;
+ let resp = self.observe(self.client.clone().get(key, None).await)?;
record_payload_bytes("get_with_revision", kvs_bytes(resp.kvs()));
let revision = resp.header().map(|h| h.revision()).unwrap_or(0);
match resp.kvs().first() {
@@ -205,7 +263,7 @@ impl EtcdStore {
pub async fn list_keys(&self, prefix: &str) -> Result<Vec<String>> {
let _t = OpTimer::new("list_keys");
let options = GetOptions::new().with_prefix().with_keys_only();
- let resp = self.client.clone().get(prefix, Some(options)).await?;
+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
record_payload_bytes("list_keys", kvs_bytes(resp.kvs()));
Ok(resp
.kvs()
@@ -219,7 +277,7 @@ impl EtcdStore {
pub async fn list_keys_with_mod_revisions(&self, prefix: &str) -> Result<Vec<(String, i64)>> {
let _t = OpTimer::new("list_keys_with_mod_revisions");
let options = GetOptions::new().with_prefix().with_keys_only();
- let resp = self.client.clone().get(prefix, Some(options)).await?;
+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
record_payload_bytes("list_keys_with_mod_revisions", kvs_bytes(resp.kvs()));
Ok(resp
.kvs()
@@ -243,7 +301,7 @@ impl EtcdStore {
) -> Result<(Vec<T>, i64)> {
let _t = OpTimer::new("list_with_revision");
let options = GetOptions::new().with_prefix();
- let resp = self.client.clone().get(prefix, Some(options)).await?;
+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
record_payload_bytes("list_with_revision", kvs_bytes(resp.kvs()));
let revision = resp.header().map(|h| h.revision()).unwrap_or(0);
let items = resp
@@ -263,7 +321,7 @@ impl EtcdStore {
) -> Result<Vec<(T, i64)>> {
let _t = OpTimer::new("list_with_mod_revisions");
let options = GetOptions::new().with_prefix();
- let resp = self.client.clone().get(prefix, Some(options)).await?;
+ let resp = self.observe(self.client.clone().get(prefix, Some(options)).await)?;
record_payload_bytes("list_with_mod_revisions", kvs_bytes(resp.kvs()));
resp.kvs()
.iter()
@@ -294,27 +352,27 @@ impl EtcdStore {
let value = serde_json::to_string(value)?;
record_payload_bytes("put", key.len() + value.len());
let options = lease_id.map(|id| PutOptions::new().with_lease(id));
- self.client.clone().put(key, value, options).await?;
+ self.observe(self.client.clone().put(key, value, options).await)?;
Ok(())
}
pub async fn delete(&self, key: &str) -> Result<()> {
let _t = OpTimer::new("delete");
- self.client.clone().delete(key, None).await?;
+ self.observe(self.client.clone().delete(key, None).await)?;
Ok(())
}
pub async fn delete_prefix(&self, prefix: &str) -> Result<()> {
let _t = OpTimer::new("delete_prefix");
let options = DeleteOptions::new().with_prefix();
- self.client.clone().delete(prefix, Some(options)).await?;
+ self.observe(self.client.clone().delete(prefix, Some(options)).await)?;
Ok(())
}
pub async fn watch(&self, prefix: &str) -> Result<WatchStream> {
let _t = OpTimer::new("watch");
let options = WatchOptions::new().with_prefix();
- let stream = self.client.clone().watch(prefix, Some(options)).await?;
+ let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;
Ok(stream)
}
@@ -330,7 +388,7 @@ impl EtcdStore {
let options = WatchOptions::new()
.with_prefix()
.with_start_revision(start_revision);
- let stream = self.client.clone().watch(prefix, Some(options)).await?;
+ let stream = self.observe(self.client.clone().watch(prefix, Some(options)).await)?;
Ok(stream)
}
@@ -340,7 +398,7 @@ impl EtcdStore {
pub async fn watch_key_from(&self, key: &str, start_revision: i64) -> Result<WatchStream> {
let _t = OpTimer::new("watch_key_from");
let options = WatchOptions::new().with_start_revision(start_revision);
- let stream = self.client.clone().watch(key, Some(options)).await?;
+ let stream = self.observe(self.client.clone().watch(key, Some(options)).await)?;
Ok(stream)
}
@@ -355,7 +413,7 @@ impl EtcdStore {
/// request side directly instead.
pub async fn txn(&self, txn: Txn) -> Result<TxnResponse> {
let _t = OpTimer::new("txn");
- Ok(self.client.clone().txn(txn).await?)
+ Ok(self.observe(self.client.clone().txn(txn).await)?)
}
/// Atomically create `key` bound to `lease_id`, only if it does not
@@ -371,7 +429,7 @@ impl EtcdStore {
value,
Some(PutOptions::new().with_lease(lease_id)),
)]);
- let resp = self.client.clone().txn(txn).await?;
+ let resp = self.observe(self.client.clone().txn(txn).await)?;
Ok(resp.succeeded())
}
@@ -379,7 +437,7 @@ impl EtcdStore {
pub async fn grant_lease(&self, ttl: i64) -> Result<i64> {
let _t = OpTimer::new("grant_lease");
- let resp = self.client.clone().lease_grant(ttl, None).await?;
+ let resp = self.observe(self.client.clone().lease_grant(ttl, None).await)?;
Ok(resp.id())
}
@@ -388,13 +446,14 @@ impl EtcdStore {
lease_id: i64,
) -> Result<(etcd_client::LeaseKeeper, etcd_client::LeaseKeepAliveStream)> {
let _t = OpTimer::new("keep_alive");
- let (keeper, stream) = self.client.clone().lease_keep_alive(lease_id).await?;
+ let (keeper, stream) =
+ self.observe(self.client.clone().lease_keep_alive(lease_id).await)?;
Ok((keeper, stream))
}
pub async fn revoke_lease(&self, lease_id: i64) -> Result<()> {
let _t = OpTimer::new("revoke_lease");
- self.client.clone().lease_revoke(lease_id).await?;
+ self.observe(self.client.clone().lease_revoke(lease_id).await)?;
Ok(())
}
@@ -419,3 +478,106 @@ pub fn parse_watch_value<T: DeserializeOwned>(
.ok_or_else(|| Error::InvalidState("watch event missing kv".to_string()))?;
serde_json::from_slice(kv.value()).map_err(Error::from)
}
+
+#[cfg(test)]
+mod member_failover_tests {
+ use std::time::Instant;
+
+ use super::*;
+
+ const LIVE_ETCD: &str = "http://localhost:2379";
+ const DEAD_ENDPOINT: &str = "http://127.0.0.1:1";
+
+ async fn store_on_dead_member(known_members: Vec<String>) -> EtcdStore {
+ let nanos = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap()
+ .as_nanos();
+ let store = EtcdStore::connect_with_member_failover(StoreConfig {
+ endpoints: vec![DEAD_ENDPOINT.to_string()],
+ prefix: format!("/test-member-failover-{nanos}/"),
+ })
+ .await
+ .expect("connect is lazy, so a dead endpoint still yields a store");
+ store
+ .failover
+ .as_ref()
+ .unwrap()
+ .set_members_for_test(known_members);
+ store
+ }
+
+ #[tokio::test]
+ async fn a_store_on_a_failing_member_moves_to_a_healthy_one() {
+ let store = store_on_dead_member(vec![LIVE_ETCD.to_string()]).await;
+
+ let deadline = Instant::now() + Duration::from_secs(15);
+ loop {
+ if store.get_raw("/probe").await.is_ok() {
+ break;
+ }
+ assert!(
+ Instant::now() < deadline,
+ "store never left the dead member"
+ );
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ assert_eq!(
+ store.failover.as_ref().unwrap().current_for_test(),
+ vec![LIVE_ETCD.to_string()]
+ );
+ }
+
+ #[tokio::test]
+ async fn a_store_without_a_member_list_reads_one_after_a_call_succeeds() {
+ let store = EtcdStore::connect_with_member_failover(StoreConfig {
+ endpoints: vec![LIVE_ETCD.to_string()],
+ prefix: "/test-member-failover-refresh/".to_string(),
+ })
+ .await
+ .unwrap();
+ let failover = store.failover.as_ref().unwrap();
+ failover.set_members_for_test(Vec::new());
+
+ store.get_raw("/probe").await.unwrap();
+
+ let deadline = Instant::now() + Duration::from_secs(5);
+ while failover.members_for_test().is_empty() {
+ assert!(Instant::now() < deadline, "member list never refreshed");
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ }
+
+ #[tokio::test]
+ async fn only_a_long_unanswered_call_switches_at_once() {
+ let store = store_on_dead_member(vec![LIVE_ETCD.to_string()]).await;
+ let failover = store.failover.as_ref().unwrap();
+
+ store.observe_member_unanswered(Duration::from_secs(1));
+ assert!(!failover.switching_for_test());
+
+ store.observe_member_unanswered(Duration::from_secs(2));
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while failover.current_for_test() != vec![LIVE_ETCD.to_string()] {
+ assert!(
+ Instant::now() < deadline,
+ "store never left the dead member"
+ );
+ tokio::time::sleep(Duration::from_millis(50)).await;
+ }
+ }
+
+ #[tokio::test]
+ async fn a_store_stays_put_when_no_other_member_answers() {
+ let store = store_on_dead_member(vec!["http://127.0.0.1:2".to_string()]).await;
+
+ for _ in 0..6 {
+ assert!(store.get_raw("/probe").await.is_err());
+ }
+ tokio::time::sleep(Duration::from_secs(4)).await;
+ assert_eq!(
+ store.failover.as_ref().unwrap().current_for_test(),
+ vec![DEAD_ENDPOINT.to_string()]
+ );
+ }
+}
diff --git a/rust/personhog-changelog-relay/src/main.rs b/rust/personhog-changelog-relay/src/main.rs
index 5eec6709a903..8c543abdab71 100644
--- a/rust/personhog-changelog-relay/src/main.rs
+++ b/rust/personhog-changelog-relay/src/main.rs
@@ -94,7 +94,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.expect("Metrics server error");
});
- let store = EtcdStore::connect(StoreConfig {
+ let store = EtcdStore::connect_with_member_failover(StoreConfig {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
})
diff --git a/rust/personhog-coordination/README.md b/rust/personhog-coordination/README.md
index f5b2eedfa791..1f7dbb57d2ac 100644
--- a/rust/personhog-coordination/README.md
+++ b/rust/personhog-coordination/README.md
@@ -94,7 +94,8 @@ Several failure modes are handled explicitly:
- **Warm provenance**: A pod records what each partition warm was installed for (serving, or a specific handoff id). Warming honors only a warm installed for the *current* handoff — one left over from an earlier ownership era is released and rebuilt, because its cache can hold values that predate writes the changelog accepted since, and a fresh warm replays only from the writer's committed offset. Without this, a returning owner whose `Released` convergence never ran could ack a warm it never performed and lose acknowledged writes — a counterexample the stateright model found and now pins.
- **Coordinator restart**: On startup, the coordinator runs `reconcile_pending_handoffs`. For every existing handoff it applies cleanup if `Complete`, otherwise calls `check_phase_advance` to nudge the state machine. This handles the case where acks were written before the coordinator was up — the ack-watch only fires on Put events, so without reconcile those acks would be invisible.
- **Reconcile tick**: Beyond the startup reconcile, the coordinator re-evaluates every in-flight handoff on a fixed interval (`reconcile_interval`, default 5 s). This is the liveness backstop for state changes that fire no watched event — e.g. a router departing (nothing watches router registrations) newly satisfying a freeze quorum. Everything the tick drives is idempotent: phase transitions use CAS and cleanup tolerates already-deleted records.
-- **Lease loss (self-fencing)**: Connection trouble and lease loss are different things. A broken or silent keepalive stream is a fact about one connection — the lease lives in etcd's replicated keyspace and stays valid until its TTL passes without renewal — so ambiguous keepalive failures rebuild the stream and retry (a fresh connection through the service reaches a healthy member) for as long as the last confirmed renewal is recent; a routine single-member restart costs some retries (`personhog_coordination_keepalive_retries_total{component}`), not a fence. The keepalive declares loss on exactly two signals: etcd answering a round with TTL ≤ 0 (authoritative — the lease is revoked or expired), or the renewal margin running out (two thirds of the TTL without a confirmed renewal — the final third is reserved so the fence completes before the coordinator can possibly treat the lease as expired; retrying past it would gamble split-brain against availability). On loss the component ends the coordination attempt immediately, skipping the graceful drain: the coordinator already considers it dead and is reassigning its partitions via the dead-owner path, so a graceful drain would only race that. A pod self-fences *locally* before anything else: every held partition is drained and, once quiesced, released (dropping its cache and serving authority — re-acquisition always re-warms). A partition whose drain cannot quiesce is deliberately **not** released — release is what unfences admissions and drops the cache, and a leaseless pod must do neither over live in-flight work — so it stays fenced and held, in-place recovery is refused, and the process restart clears it by death. A zombie owner therefore never keeps serving partitions the protocol has reassigned beyond the window that restart closes. Lease loss is observed at every await the session makes while holding a lease — the attempt select, the backoff nap between failed attempts, and the graceful drain all race the keepalive — so once the keepalive declares loss, the fence is never deferred by supervisor state. Every other session end fences too — budget exhaustion and shutdown revoke the lease, which deletes the registration instantly, so the local fence runs first (a no-op after a completed graceful drain) and the dead-owner path never meets a still-serving owner. On lease loss the fence's drain budget is the runway the keepalive margin reserved (a third of the TTL), not the full graceful drain timeout; overshoot poisons recovery and the process restart clears stragglers by death.
+- **Lease loss (self-fencing)**: Connection trouble and lease loss are different things. A broken or silent keepalive stream is a fact about one connection — the lease lives in etcd's replicated keyspace and stays valid until its TTL passes without renewal — so ambiguous keepalive failures rebuild the stream and retry (the store moves to another member when its own keeps failing; see `assignment_coordination::failover`) for as long as the last confirmed renewal is recent; a routine single-member restart costs some retries (`personhog_coordination_keepalive_retries_total{component}`), not a fence. The keepalive declares loss on exactly two signals: etcd answering a round with TTL ≤ 0 (authoritative — the lease is revoked or expired), or the renewal margin running out (two thirds of the TTL without a confirmed renewal — the final third is reserved so the fence completes before the coordinator can possibly treat the lease as expired; retrying past it would gamble split-brain against availability). On loss the component ends the coordination attempt immediately, skipping the graceful drain: the coordinator already considers it dead and is reassigning its partitions via the dead-owner path, so a graceful drain would only race that. A pod self-fences *locally* before anything else: every held partition is drained and, once quiesced, released (dropping its cache and serving authority — re-acquisition always re-warms). A partition whose drain cannot quiesce is deliberately **not** released — release is what unfences admissions and drops the cache, and a leaseless pod must do neither over live in-flight work — so it stays fenced and held, in-place recovery is refused, and the process restart clears it by death. A zombie owner therefore never keeps serving partitions the protocol has reassigned beyond the window that restart closes. Lease loss is observed at every await the session makes while holding a lease — the attempt select, the backoff nap between failed attempts, and the graceful drain all race the keepalive — so once the keepalive declares loss, the fence is never deferred by supervisor state. Every other session end fences too — budget exhaustion and shutdown revoke the lease, which deletes the registration instantly, so the local fence runs first (a no-op after a completed graceful drain) and the dead-owner path never meets a still-serving owner. On lease loss the fence's drain budget is the runway the keepalive margin reserved (a third of the TTL), not the full graceful drain timeout; overshoot poisons recovery and the process restart clears stragglers by death.
+- **etcd member failover**: Each process keeps one connection to whichever etcd member it reached first. Three member-level failures in a row (Unavailable, DeadlineExceeded, a broken connection), or one call left unanswered for at least 2 s, start a switch: the store probes the other members from its cached member list with a linearizable read and moves new calls to the first that answers. Requests carry etcd's require-leader flag, so a member without a leader rejects them at once. Watches and keepalive streams already open stay on their connection until it closes or etcd cancels them. A renewal waits up to the heartbeat interval for an answer, so a member that stops answering without closing its connections still costs that pod one renewal margin, and with it the pod's partitions.
- **Supervised coordination runs (etcd resilience)**: A failed coordination attempt — a broken watch stream, a failed etcd write, an exhausted reconcile budget — no longer kills the process; the run supervisors tear the attempt down cleanly, back off exponentially, and rebuild through the same bootstrap that recovers a restarted process, while the data plane keeps serving. The two participant classes recover *differently*, on purpose. A **router** sheds its registration on every failed attempt (its registration is a freeze-quorum vote, and a registered but non-acking router would stall every quorum until the phase deadline); serving through its gap is safe because handoffs created while it is unregistered exclude it from their quorums and the leader-side fences reject its stale forwards. A **pod** does the opposite: its lease session (grant, register, heartbeat) lives *above* the attempt loop and survives attempt failures, so the coordinator keeps seeing a live pod and never runs the dead-owner path against a process that is still serving — the invariant is that serving authority is held exactly as long as the lease-backed registration behind it. This matters because the dead-owner path has no fence: its safety rests entirely on a deregistered owner being unable to produce, so a pod must never voluntarily deregister while it still serves. Only lease loss ends a pod session, self-fencing locally (every held partition released; re-acquisition re-warms) before a fresh session registers anew. A consecutive-failure budget (`ROUTER_RUN_RETRY_BUDGET`, default 10; same default on pods) keeps a genuinely wedged participant on the process-restart path; an attempt that made real progress — a convergence applied, a reconcile pass completed — resets the count, and the reset is keyed on measured progress rather than elapsed time because a time threshold silently exempts every failure detector slower than it (the reconcile budget and the stall watchdog both take a minute to fire), and `personhog_coordination_run_restarts_total{component}` counts every rebuild.
### Concurrency and atomicity
diff --git a/rust/personhog-coordination/src/error.rs b/rust/personhog-coordination/src/error.rs
index 6c1198cd838a..1685c896016d 100644
--- a/rust/personhog-coordination/src/error.rs
+++ b/rust/personhog-coordination/src/error.rs
@@ -35,6 +35,14 @@ impl Error {
Self::Store(assignment_coordination::error::Error::LeadershipLost)
}
+ /// The etcd error underneath, when etcd is what failed.
+ pub fn etcd_error(&self) -> Option<&etcd_client::Error> {
+ match self {
+ Self::Store(assignment_coordination::error::Error::Etcd(e)) => Some(e),
+ _ => None,
+ }
+ }
+
/// Whether losing the election lease is what ended this term. An
/// abdication is the protocol working — a successor takes over and
/// reconciles — so it is not evidence that this process is unhealthy.
diff --git a/rust/personhog-coordination/src/util.rs b/rust/personhog-coordination/src/util.rs
index 1d8f69377edd..545556080936 100644
--- a/rust/personhog-coordination/src/util.rs
+++ b/rust/personhog-coordination/src/util.rs
@@ -148,15 +148,19 @@ pub async fn run_lease_keepalive(
if left.is_zero() {
return Err(margin_exhausted());
}
- let attempt = tokio::time::timeout(left.min(interval), store.keep_alive(lease_id));
+ let wait = left.min(interval);
+ let attempt = tokio::time::timeout(wait, store.keep_alive(lease_id));
let failure = tokio::select! {
_ = cancel.cancelled() => return Ok(()),
r = attempt => match r {
Ok(Ok(pair)) => break pair,
Ok(Err(e)) => e,
- Err(_) => Error::invalid_state(
- "keepalive stream establishment timed out".to_string(),
- ),
+ Err(_) => {
+ store.inner().observe_member_unanswered(wait);
+ Error::invalid_state(
+ "keepalive stream establishment timed out".to_string(),
+ )
+ }
},
};
counter!(
@@ -219,12 +223,24 @@ pub async fn run_lease_keepalive(
// catches those.
crate::store::count_call("keep_alive_renewal");
let _renewal = assignment_coordination::store::OpTimer::new("keep_alive_renewal");
- match tokio::time::timeout(left.min(interval), round).await {
- Ok(r) => r,
- Err(_) => Err(Error::invalid_state(format!(
- "keepalive round unanswered within {:?}",
- left.min(interval)
- ))),
+ let wait = left.min(interval);
+ match tokio::time::timeout(wait, round).await {
+ Ok(r) => {
+ match &r {
+ Ok(_) => store.inner().observe_member_outcome(Ok(())),
+ Err(e) => match e.etcd_error() {
+ Some(etcd) => store.inner().observe_member_outcome(Err(etcd)),
+ None => store.inner().observe_member_failure(),
+ },
+ }
+ r
+ }
+ Err(_) => {
+ store.inner().observe_member_unanswered(wait);
+ Err(Error::invalid_state(format!(
+ "keepalive round unanswered within {wait:?}"
+ )))
+ }
}
};
match outcome {
diff --git a/rust/personhog-identity/src/main.rs b/rust/personhog-identity/src/main.rs
index 87ffde0cada4..8446092f723c 100644
--- a/rust/personhog-identity/src/main.rs
+++ b/rust/personhog-identity/src/main.rs
@@ -148,7 +148,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// The delete saga groups its fence calls by the leaders' partition
// count, which lives in etcd; a service that cannot read it would
// batch wrong, so refuse to start instead.
- let etcd_store = EtcdStore::connect(StoreConfig {
+ let etcd_store = EtcdStore::connect_with_member_failover(StoreConfig {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
})
diff --git a/rust/personhog-leader/src/main.rs b/rust/personhog-leader/src/main.rs
index ba06cb9c5e3b..db0c09d6bfbb 100644
--- a/rust/personhog-leader/src/main.rs
+++ b/rust/personhog-leader/src/main.rs
@@ -361,7 +361,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
};
- let etcd_store = EtcdStore::connect(etcd_config)
+ let etcd_store = EtcdStore::connect_with_member_failover(etcd_config)
.await
.expect("Failed to connect to etcd");
let store = Arc::new(PersonhogStore::new(etcd_store));
diff --git a/rust/personhog-router/src/main.rs b/rust/personhog-router/src/main.rs
index c19739c3f5ab..dfd599040890 100644
--- a/rust/personhog-router/src/main.rs
+++ b/rust/personhog-router/src/main.rs
@@ -253,7 +253,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
};
- let etcd_store = EtcdStore::connect(etcd_config)
+ let etcd_store = EtcdStore::connect_with_member_failover(etcd_config)
.await
.expect("Failed to connect to etcd");
let store = Arc::new(PersonhogStore::new(etcd_store));
diff --git a/rust/personhog-writer/src/main.rs b/rust/personhog-writer/src/main.rs
index 0e10d22c0618..659541589f8b 100644
--- a/rust/personhog-writer/src/main.rs
+++ b/rust/personhog-writer/src/main.rs
@@ -55,7 +55,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!("Upsert concurrency: {}", config.upsert_concurrency);
tracing::info!("Metrics port: {}", config.metrics_port);
- let store = EtcdStore::connect(StoreConfig {
+ let store = EtcdStore::connect_with_member_failover(StoreConfig {
endpoints: config.etcd_endpoint_list(),
prefix: config.etcd_prefix.clone(),
})