:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
#![allow(dead_code)]
use personhog_coordination::authority::AuthorityClock;
use std::collections::{HashMap, HashSet};
use std::future::{pending, Future};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex as StdMutex, RwLock as StdRwLock};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use tokio::io::copy_bidirectional;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{Mutex, Notify, RwLock};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use assignment_coordination::store::{EtcdStore, StoreConfig};
use personhog_coordination::coordinator::{Coordinator, CoordinatorConfig};
use personhog_coordination::error::Result;
use personhog_coordination::pod::{HandoffHandler, PodConfig, PodHandle};
use personhog_coordination::routing_table::{RoutingTable, RoutingTableConfig, StashHandler};
use personhog_coordination::store::PersonhogStore;
use personhog_coordination::strategy::AssignmentStrategy;
pub const ETCD_ENDPOINT: &str = "http://localhost:2379";
pub const WAIT_TIMEOUT: Duration = Duration::from_secs(10);
pub const POLL_INTERVAL: Duration = Duration::from_millis(100);
pub async fn test_store(test_name: &str) -> Arc<PersonhogStore> {
test_store_with_prefix(test_name).await.0
}
/// Like `test_store`, but also returns the generated etcd prefix so tests
/// can inspect or manipulate raw keys (e.g. revoking a component's lease).
pub async fn test_store_with_prefix(test_name: &str) -> (Arc<PersonhogStore>, String) {
let prefix = format!("/test-{}-{}/", test_name, uuid::Uuid::new_v4());
let config = StoreConfig {
endpoints: vec![ETCD_ENDPOINT.to_string()],
prefix: prefix.clone(),
};
let inner = EtcdStore::connect(config)
.await
.expect("failed to connect to etcd");
(Arc::new(PersonhogStore::new(inner)), prefix)
}
/// Revoke the etcd lease attached to `key`, simulating a component whose
/// lease expired (etcd partition, missed heartbeats) while the process
/// itself is still running.
pub async fn revoke_lease_of_key(key: &str) {
let mut client = etcd_client::Client::connect([ETCD_ENDPOINT], None)
.await
.expect("connect raw etcd client");
let resp = client.get(key, None).await.expect("get key");
let kv = resp
.kvs()
.first()
.unwrap_or_else(|| panic!("key {key} not found"));
let lease_id = kv.lease();
assert_ne!(lease_id, 0, "key {key} has no lease attached");
client.lease_revoke(lease_id).await.expect("revoke lease");
}
pub async fn wait_for_condition<F, Fut>(timeout: Duration, interval: Duration, f: F)
where
F: Fn() -> Fut,
Fut: Future<Output = bool>,
{
let start = Instant::now();
while start.elapsed() < timeout {
if f().await {
return;
}
tokio::time::sleep(interval).await;
}
panic!("condition not met within {timeout:?}");
}
/// `wait_for_condition` with a description of what is being waited on.
///
/// A bare timeout reports "condition not met", which names nothing —
/// and for tests whose whole assertion *is* the wait, that is the entire
/// failure message. Worth using wherever the timeout is the assertion
/// rather than a setup step. (`#[track_caller]` would be the zero-churn
/// answer, but it is a no-op on async fns.)
#[allow(dead_code)]
pub async fn wait_for_condition_named<F, Fut>(
timeout: Duration,
interval: Duration,
what: &str,
f: F,
) where
F: Fn() -> Fut,
Fut: Future<Output = bool>,
{
let start = Instant::now();
while start.elapsed() < timeout {
if f().await {
return;
}
tokio::time::sleep(interval).await;
}
panic!("timed out after {timeout:?} waiting for {what}");
}
// ── Component builders ──────────────────────────────────────────
pub fn start_coordinator(
store: Arc<PersonhogStore>,
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
start_coordinator_named(store, "coordinator-0", 10, strategy, cancel)
}
pub fn start_coordinator_named(
store: Arc<PersonhogStore>,
name: &str,
leader_lease_ttl: i64,
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
start_coordinator_with_deadline(
store,
name,
leader_lease_ttl,
Duration::from_secs(86_400),
strategy,
cancel,
)
}
pub fn start_coordinator_with_deadline(
store: Arc<PersonhogStore>,
name: &str,
leader_lease_ttl: i64,
handoff_deadline: Duration,
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
let config = coordinator_config(name, leader_lease_ttl, handoff_deadline);
start_coordinator_with_config(store, config, strategy, cancel)
}
pub fn coordinator_config(
name: &str,
leader_lease_ttl: i64,
handoff_deadline: Duration,
) -> CoordinatorConfig {
let keepalive_secs = (leader_lease_ttl as u64 / 3).max(1);
CoordinatorConfig {
name: name.to_string(),
leader_lease_ttl,
keepalive_interval: Duration::from_secs(keepalive_secs),
// Short enough that a failover never waits on the leader-key
// watch alone.
standby_poll_interval: Duration::from_millis(500),
run_retry_backoff: Duration::from_millis(10),
backoff_decay_window: Duration::from_secs(300),
rebalance_debounce_interval: Duration::from_millis(100),
reconcile_interval: Duration::from_millis(500),
// Callers default this to a day: these tests deliberately
// park handoffs mid-phase to assert what the protocol does
// with them, and a live deadline would replace the state
// under test. The cancellation test passes a short one
// explicitly.
handoff_deadline,
warming_deadline: Duration::from_secs(86_400),
max_txn_ops: 128,
migration_pre_flip_deadline: Duration::from_secs(86_400),
migration_debounce_interval: Duration::from_millis(10),
}
}
pub fn start_coordinator_with_config(
store: Arc<PersonhogStore>,
config: CoordinatorConfig,
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
let coordinator = Coordinator::new(store, config, strategy, None);
let token = cancel.child_token();
tokio::spawn(async move {
coordinator.run(token).await;
Ok(())
})
}
pub struct PodHandles {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
pub join_handle: Option<JoinHandle<Result<()>>>,
}
pub fn start_pod(store: Arc<PersonhogStore>, name: &str, cancel: CancellationToken) -> PodHandles {
start_pod_with_lease_ttl(store, name, 10, cancel)
}
pub fn start_pod_with_lease_ttl(
store: Arc<PersonhogStore>,
name: &str,
lease_ttl: i64,
cancel: CancellationToken,
) -> PodHandles {
start_pod_with_address(store, name, lease_ttl, None, cancel)
}
pub fn start_pod_with_address(
store: Arc<PersonhogStore>,
name: &str,
lease_ttl: i64,
advertise_address: Option<String>,
cancel: CancellationToken,
) -> PodHandles {
let heartbeat_secs = (lease_ttl as u64 / 3).max(1);
let (handler, events) = MockHandoffHandler::new();
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs(heartbeat_secs),
advertise_address,
// Parked: event-driven tests assert exact handler-call
// sequences a live reconcile pass would duplicate.
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// A handler whose `resume_partition` fails a fixed number of times
/// before succeeding — the shape of a resume that has to take broker
/// state and hits a transient error.
pub struct FlakyResumeHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
remaining_failures: AtomicUsize,
}
#[async_trait]
impl HandoffHandler for FlakyResumeHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
if self.remaining_failures.load(Ordering::SeqCst) > 0 {
self.remaining_failures.fetch_sub(1, Ordering::SeqCst);
return Err(personhog_coordination::error::Error::invalid_state(
"resume failed (test)",
));
}
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Start a pod whose first `failures` resume attempts fail.
pub fn start_pod_with_flaky_resume(
store: Arc<PersonhogStore>,
name: &str,
failures: usize,
cancel: CancellationToken,
) -> PodHandles {
let events = Arc::new(Mutex::new(Vec::new()));
let handler = FlakyResumeHandler {
events: Arc::clone(&events),
remaining_failures: AtomicUsize::new(failures),
};
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl: 10,
heartbeat_interval: Duration::from_secs(3),
advertise_address: None,
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// A handler whose first `failures` release attempts fail — the shape of
/// a release that has to give back broker state and hits a transient
/// error.
pub struct FlakyReleaseHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
remaining_failures: AtomicUsize,
}
#[async_trait]
impl HandoffHandler for FlakyReleaseHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
if self.remaining_failures.load(Ordering::SeqCst) > 0 {
self.remaining_failures.fetch_sub(1, Ordering::SeqCst);
return Err(personhog_coordination::error::Error::invalid_state(
"release failed (test)",
));
}
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Start a pod whose first `failures` release attempts fail.
pub fn start_pod_with_flaky_release(
store: Arc<PersonhogStore>,
name: &str,
failures: usize,
cancel: CancellationToken,
) -> PodHandles {
let events = Arc::new(Mutex::new(Vec::new()));
let handler = FlakyReleaseHandler {
events: Arc::clone(&events),
remaining_failures: AtomicUsize::new(failures),
};
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl: 10,
heartbeat_interval: Duration::from_secs(3),
advertise_address: None,
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// A handler whose `drain_partition_inflight` always fails for one
/// partition — the shape of a drain that cannot quiesce whatever the
/// pod does.
pub struct StuckDrainHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
stuck: u32,
}
#[async_trait]
impl HandoffHandler for StuckDrainHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
if partition == self.stuck {
self.events
.lock()
.await
.push(HandoffEvent::DrainFailed(partition));
return Err(personhog_coordination::error::Error::invalid_state(
format!("drain refuses for partition {partition}"),
));
}
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Start a pod whose drain refuses for exactly one partition.
pub fn start_pod_with_stuck_drain(
store: Arc<PersonhogStore>,
name: &str,
stuck: u32,
lease_ttl: i64,
cancel: CancellationToken,
) -> PodHandles {
let events = Arc::new(Mutex::new(Vec::new()));
let handler = StuckDrainHandler {
events: Arc::clone(&events),
stuck,
};
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs((lease_ttl / 5).max(1) as u64),
advertise_address: None,
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// A handler whose `drain_partition_inflight` hangs forever for one
/// partition — the shape of in-flight work that never quiesces, which
/// only the self-fence's own timeout can end.
pub struct HangingDrainHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
hung: u32,
}
#[async_trait]
impl HandoffHandler for HangingDrainHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
if partition == self.hung {
pending::<()>().await;
}
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Start a pod whose drain hangs forever for exactly one partition.
pub fn start_pod_with_hanging_drain(
store: Arc<PersonhogStore>,
name: &str,
hung: u32,
lease_ttl: i64,
cancel: CancellationToken,
) -> PodHandles {
let events = Arc::new(Mutex::new(Vec::new()));
let handler = HangingDrainHandler {
events: Arc::clone(&events),
hung,
};
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs((lease_ttl / 5).max(1) as u64),
advertise_address: None,
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// A handler whose `release_partition` always fails, recording every
/// partition it was asked to give up before it does.
///
/// Failing for *every* partition is what makes the attempt log
/// order-independent: the release loop walks a `HashSet`, so a handler
/// that failed for only one would prove nothing when that one happened to
/// be walked last.
pub struct FailingReleaseHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
attempts: Arc<Mutex<Vec<u32>>>,
}
#[async_trait]
impl HandoffHandler for FailingReleaseHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.attempts.lock().await.push(partition);
Err(personhog_coordination::error::Error::invalid_state(
format!("release refuses for partition {partition}"),
))
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Start a pod whose every release fails, alongside the log of which
/// partitions it was asked to release.
pub fn start_pod_with_failing_release(
store: Arc<PersonhogStore>,
name: &str,
lease_ttl: i64,
cancel: CancellationToken,
) -> (PodHandles, Arc<Mutex<Vec<u32>>>) {
let events = Arc::new(Mutex::new(Vec::new()));
let attempts = Arc::new(Mutex::new(Vec::new()));
let handler = FailingReleaseHandler {
events: Arc::clone(&events),
attempts: Arc::clone(&attempts),
};
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs((lease_ttl / 5).max(1) as u64),
advertise_address: None,
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
(
PodHandles {
events,
join_handle: Some(join_handle),
},
attempts,
)
}
/// Start a pod whose warm_partition blocks forever. Useful for testing
/// crashes during the Warming phase.
pub fn start_pod_blocking(
store: Arc<PersonhogStore>,
name: &str,
lease_ttl: i64,
cancel: CancellationToken,
) -> PodHandles {
let heartbeat_secs = (lease_ttl as u64 / 3).max(1);
let (handler, events) = BlockingHandoffHandler::new();
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs(heartbeat_secs),
// Parked: event-driven tests assert exact handler-call
// sequences a live reconcile pass would duplicate.
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
/// Coordinator whose reconcile tick and phase deadlines are parked, so a
/// test can prove an advancement was event-driven rather than rescued by
/// the periodic backstop.
pub fn start_coordinator_reconcile_parked(
store: Arc<PersonhogStore>,
strategy: Arc<dyn AssignmentStrategy>,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
let coordinator = Coordinator::new(
store,
CoordinatorConfig {
reconcile_interval: Duration::from_secs(86_400),
handoff_deadline: Duration::from_secs(86_400),
warming_deadline: Duration::from_secs(86_400),
..Default::default()
},
strategy,
None,
);
let token = cancel.child_token();
tokio::spawn(async move {
coordinator.run(token).await;
Ok(())
})
}
pub fn start_coordinator_with_debounce(
store: Arc<PersonhogStore>,
strategy: Arc<dyn AssignmentStrategy>,
debounce_interval: Duration,
cancel: CancellationToken,
) -> JoinHandle<Result<()>> {
let coordinator = Coordinator::new(
store,
CoordinatorConfig {
rebalance_debounce_interval: debounce_interval,
..Default::default()
},
strategy,
None,
);
let token = cancel.child_token();
tokio::spawn(async move {
coordinator.run(token).await;
Ok(())
})
}
pub fn start_pod_slow(
store: Arc<PersonhogStore>,
name: &str,
warm_delay: Duration,
cancel: CancellationToken,
) -> PodHandles {
let (handler, events) = SlowHandoffHandler::new(warm_delay);
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
// Parked: event-driven tests assert exact handler-call
// sequences a live reconcile pass would duplicate.
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
PodHandles {
events,
join_handle: Some(join_handle),
}
}
pub struct RouterHandles {
pub events: Arc<Mutex<Vec<CutoverEvent>>>,
pub addresses: Arc<StdRwLock<HashMap<String, String>>>,
pub table: Arc<RwLock<HashMap<u32, String>>>,
pub join_handle: Option<JoinHandle<Result<()>>>,
}
pub fn start_router(
store: Arc<PersonhogStore>,
name: &str,
cancel: CancellationToken,
) -> RouterHandles {
start_router_with_lease_ttl(store, name, 10, cancel)
}
pub fn start_router_with_lease_ttl(
store: Arc<PersonhogStore>,
name: &str,
lease_ttl: i64,
cancel: CancellationToken,
) -> RouterHandles {
let heartbeat_secs = (lease_ttl as u64 / 3).max(1);
let (handler, events) = MockCutoverHandler::new();
let router = RoutingTable::new(
store,
RoutingTableConfig {
router_name: name.to_string(),
lease_ttl,
heartbeat_interval: Duration::from_secs(heartbeat_secs),
// Parked: event-driven tests assert exact handler-call
// sequences, which a live reconcile pass would re-assert
// nondeterministically. Reconcile-specific tests pass a
// short interval explicitly.
reconcile_interval: Duration::from_secs(86_400),
..RoutingTableConfig::default()
},
);
let table = router.table_handle();
let addresses = router.addresses_handle();
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { router.run(token, Arc::new(handler)).await });
RouterHandles {
events,
addresses,
table,
join_handle: Some(join_handle),
}
}
/// Connect a store to an arbitrary endpoint (e.g. a `FlakyProxy`) under
/// an explicit prefix, so a component under test can run through a
/// fault-injected connection while the test asserts against a direct one.
pub async fn store_at(endpoint: &str, prefix: &str) -> Arc<PersonhogStore> {
let config = StoreConfig {
endpoints: vec![endpoint.to_string()],
prefix: prefix.to_string(),
};
let inner = EtcdStore::connect(config)
.await
.expect("failed to connect store");
Arc::new(PersonhogStore::new(inner))
}
/// A byte-forwarding TCP proxy for fault-injecting a component's etcd
/// connection: `sever` breaks every live connection (in-flight streams
/// error; reconnects still succeed), `set_blackholed(true)` also kills
/// new connections on accept, so recovery is impossible until it is
/// lifted, and `set_hanging(true)` accepts them and then does nothing.
///
/// The three are different failures, and the difference matters. Severed
/// and blackholed both fail *fast* — the client gets a reset and an error
/// it can act on. Hanging is the silent partition: the connection is
/// established, the request goes out, and no answer ever comes, so a
/// caller with no timeout of its own waits for the transport rather than
/// for anything it chose.
pub struct FlakyProxy {
/// Endpoint URL to hand to `store_at`.
pub endpoint: String,
conns: Arc<StdMutex<Vec<tokio::task::JoinHandle<()>>>>,
/// Accepted-and-parked sockets, held open so the peer sees a live
/// connection that never answers. Dropping them would turn the hang
/// into a reset, which is the failure this mode exists to not be.
parked: Arc<StdMutex<Vec<TcpStream>>>,
blackholed: Arc<AtomicBool>,
hanging: Arc<AtomicBool>,
accepted: Arc<AtomicUsize>,
listener: tokio::task::JoinHandle<()>,
}
impl FlakyProxy {
pub async fn start(upstream: &'static str) -> Self {
let socket = TcpListener::bind("127.0.0.1:0").await.expect("bind proxy");
let endpoint = format!("http://{}", socket.local_addr().expect("proxy addr"));
let conns: Arc<StdMutex<Vec<tokio::task::JoinHandle<()>>>> =
Arc::new(StdMutex::new(Vec::new()));
let parked: Arc<StdMutex<Vec<TcpStream>>> = Arc::new(StdMutex::new(Vec::new()));
let blackholed = Arc::new(AtomicBool::new(false));
let hanging = Arc::new(AtomicBool::new(false));
let accepted = Arc::new(AtomicUsize::new(0));
let accepted_bg = Arc::clone(&accepted);
let conns_bg = Arc::clone(&conns);
let parked_bg = Arc::clone(&parked);
let blackholed_bg = Arc::clone(&blackholed);
let hanging_bg = Arc::clone(&hanging);
let listener = tokio::spawn(async move {
loop {
let Ok((mut client, _)) = socket.accept().await else {
return;
};
accepted_bg.fetch_add(1, Ordering::SeqCst);
if blackholed_bg.load(Ordering::SeqCst) {
drop(client);
continue;
}
if hanging_bg.load(Ordering::SeqCst) {
// Held rather than dropped: the peer must see a
// connection that is open and silent, not one that
// was refused.
parked_bg.lock().unwrap().push(client);
continue;
}
let pump = tokio::spawn(async move {
let Ok(mut upstream_conn) = TcpStream::connect(upstream).await else {
return;
};
drop(copy_bidirectional(&mut client, &mut upstream_conn).await);
});
conns_bg.lock().unwrap().push(pump);
}
});
Self {
endpoint,
conns,
parked,
blackholed,
hanging,
accepted,
listener,
}
}
/// How many connections the proxy has accepted since it started.
///
/// A blackholed proxy drops each one immediately, so a client that
/// keeps retrying keeps climbing this — which is what makes "it is
/// still trying" a fact a test can wait on rather than a duration it
/// hopes is long enough.
pub fn accepted(&self) -> usize {
self.accepted.load(Ordering::SeqCst)
}
/// Break every live connection; the streams running over them error
/// out. New connections still succeed unless blackholed.
pub fn sever(&self) {
for pump in self.conns.lock().unwrap().drain(..) {
pump.abort();
}
}
pub fn set_blackholed(&self, blackholed: bool) {
self.blackholed.store(blackholed, Ordering::SeqCst);
}
/// Accept new connections and then answer nothing on them.
///
/// The silent partition: a caller that sets no timeout of its own
/// waits for the transport to give up rather than for a deadline it
/// chose, which is what makes an unraced etcd call outlast the budget
/// its component was given.
pub fn set_hanging(&self, hanging: bool) {
self.hanging.store(hanging, Ordering::SeqCst);
}
}
impl Drop for FlakyProxy {
fn drop(&mut self) {
self.listener.abort();
self.sever();
self.parked.lock().unwrap().clear();
}
}
// ── Mock handlers ───────────────────────────────────────────────
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HandoffEvent {
Drained(u32),
/// A drain attempt that returned an error — pushed by handlers whose
/// failures are the scenario, so tests can sequence on the attempt
/// having happened rather than racing the watch.
DrainFailed(u32),
Warmed(u32),
Released(u32),
Resumed(u32),
}
pub struct MockHandoffHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
}
impl MockHandoffHandler {
pub fn new() -> (Self, Arc<Mutex<Vec<HandoffEvent>>>) {
let events = Arc::new(Mutex::new(Vec::new()));
(
Self {
events: Arc::clone(&events),
},
events,
)
}
}
#[async_trait]
impl HandoffHandler for MockHandoffHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// A handoff handler that blocks forever on warm_partition.
/// Simulates a pod that crashes before warming completes.
pub struct BlockingHandoffHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
}
impl BlockingHandoffHandler {
pub fn new() -> (Self, Arc<Mutex<Vec<HandoffEvent>>>) {
let events = Arc::new(Mutex::new(Vec::new()));
(
Self {
events: Arc::clone(&events),
},
events,
)
}
}
#[async_trait]
impl HandoffHandler for BlockingHandoffHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, _partition: u32) -> Result<()> {
// Block forever — simulates a slow warm that never completes
pending().await
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
/// Per-partition gates for `GatedWarmHandler`: a warm parks until its
/// partition's gate opens, giving tests deterministic control over which
/// warms are in flight at once. A gate stays open once opened.
#[derive(Clone, Default)]
pub struct WarmGates {
open: Arc<StdMutex<HashSet<u32>>>,
notify: Arc<Notify>,
}
impl WarmGates {
pub fn open(&self, partition: u32) {
self.open.lock().unwrap().insert(partition);
self.notify.notify_waiters();
}
async fn wait_open(&self, partition: u32) {
loop {
let notified = self.notify.notified();
if self.open.lock().unwrap().contains(&partition) {
return;
}
notified.await;
}
}
}
/// A handoff handler whose warms park on per-partition gates while
/// recording how many warms run concurrently — in total and for the same
/// partition. Drives the convergence-lane tests: the gates prove
/// cross-partition parallelism and the warm-slot bound, and the
/// same-partition maximum pins single-flight convergence.
pub struct GatedWarmHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
pub gates: WarmGates,
pub warms_in_flight: Arc<StdMutex<HashMap<u32, usize>>>,
pub max_concurrent_warms: Arc<AtomicUsize>,
pub max_concurrent_same_partition: Arc<AtomicUsize>,
}
impl GatedWarmHandler {
pub fn new() -> Self {
Self {
events: Arc::new(Mutex::new(Vec::new())),
gates: WarmGates::default(),
warms_in_flight: Arc::new(StdMutex::new(HashMap::new())),
max_concurrent_warms: Arc::new(AtomicUsize::new(0)),
max_concurrent_same_partition: Arc::new(AtomicUsize::new(0)),
}
}
}
#[async_trait]
impl HandoffHandler for GatedWarmHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
{
let mut in_flight = self.warms_in_flight.lock().unwrap();
*in_flight.entry(partition).or_insert(0) += 1;
let total: usize = in_flight.values().sum();
self.max_concurrent_warms.fetch_max(total, Ordering::SeqCst);
self.max_concurrent_same_partition
.fetch_max(in_flight[&partition], Ordering::SeqCst);
}
self.gates.wait_open(partition).await;
*self
.warms_in_flight
.lock()
.unwrap()
.get_mut(&partition)
.unwrap() -= 1;
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
pub struct GatedPodHandles {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
pub gates: WarmGates,
pub warms_in_flight: Arc<StdMutex<HashMap<u32, usize>>>,
pub max_concurrent_warms: Arc<AtomicUsize>,
pub max_concurrent_same_partition: Arc<AtomicUsize>,
pub join_handle: JoinHandle<Result<()>>,
}
/// Start a pod backed by a `GatedWarmHandler` with the given warm
/// concurrency bound.
pub fn start_pod_gated(
store: Arc<PersonhogStore>,
name: &str,
warm_concurrency: usize,
cancel: CancellationToken,
) -> GatedPodHandles {
let handler = GatedWarmHandler::new();
let events = Arc::clone(&handler.events);
let gates = handler.gates.clone();
let warms_in_flight = Arc::clone(&handler.warms_in_flight);
let max_concurrent_warms = Arc::clone(&handler.max_concurrent_warms);
let max_concurrent_same_partition = Arc::clone(&handler.max_concurrent_same_partition);
let pod = PodHandle::new(
store,
PodConfig {
pod_name: name.to_string(),
warm_concurrency,
// Parked: event-driven tests assert exact handler-call
// sequences a live reconcile pass would duplicate.
reconcile_interval: Duration::from_secs(86_400),
..Default::default()
},
Arc::new(handler),
None,
Arc::new(AuthorityClock::unclaimed()),
);
let token = cancel.child_token();
let join_handle = tokio::spawn(async move { pod.run(token).await });
GatedPodHandles {
events,
gates,
warms_in_flight,
max_concurrent_warms,
max_concurrent_same_partition,
join_handle,
}
}
/// A handoff handler that adds a configurable delay to warm_partition.
/// Simulates a pod that takes time to warm its cache.
pub struct SlowHandoffHandler {
pub events: Arc<Mutex<Vec<HandoffEvent>>>,
pub warm_delay: Duration,
}
impl SlowHandoffHandler {
pub fn new(warm_delay: Duration) -> (Self, Arc<Mutex<Vec<HandoffEvent>>>) {
let events = Arc::new(Mutex::new(Vec::new()));
(
Self {
events: Arc::clone(&events),
warm_delay,
},
events,
)
}
}
#[async_trait]
impl HandoffHandler for SlowHandoffHandler {
async fn drain_partition_inflight(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Drained(partition));
Ok(())
}
async fn warm_partition(&self, partition: u32) -> Result<()> {
tokio::time::sleep(self.warm_delay).await;
self.events
.lock()
.await
.push(HandoffEvent::Warmed(partition));
Ok(())
}
async fn release_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Released(partition));
Ok(())
}
async fn resume_partition(&self, partition: u32) -> Result<()> {
self.events
.lock()
.await
.push(HandoffEvent::Resumed(partition));
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CutoverEvent {
StashBegan { partition: u32, new_owner: String },
StashDrained { partition: u32, target: String },
}
pub struct MockCutoverHandler {
pub events: Arc<Mutex<Vec<CutoverEvent>>>,
}
impl MockCutoverHandler {
pub fn new() -> (Self, Arc<Mutex<Vec<CutoverEvent>>>) {
let events = Arc::new(Mutex::new(Vec::new()));
(
Self {
events: Arc::clone(&events),
},
events,
)
}
}
#[async_trait]
impl StashHandler for MockCutoverHandler {
async fn begin_stash(&self, partition: u32, new_owner: &str) -> Result<()> {
self.events.lock().await.push(CutoverEvent::StashBegan {
partition,
new_owner: new_owner.to_string(),
});
Ok(())
}
async fn drain_stash(
&self,
partition: u32,
target: &str,
_cancel: CancellationToken,
) -> Result<()> {
self.events.lock().await.push(CutoverEvent::StashDrained {
partition,
target: target.to_string(),
});
Ok(())
}
}