:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::{Duration, Instant};
use dashmap::mapref::entry::Entry;
use dashmap::DashMap;
use metrics::{counter, gauge, histogram};
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use crate::config::Config;
use crate::debug_recorder::{record_if, DebugEventKind, DebugRecorder, WorkerHealthSnapshot};
const PASSIVE_WINDOW_MAX_ENTRIES: usize = 10_000;
/// Stable identity for a worker: its base HTTP URL (e.g. `http://10.0.0.1:9001`).
/// Used as the key for health, placement, and in-flight load so the worker
/// set can change at runtime without the index-shifting hazards of a `Vec`.
pub type WorkerId = Arc<str>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WorkerState {
Healthy,
Degraded,
Unhealthy,
}
impl WorkerState {
const ALL: [WorkerState; 3] = [
WorkerState::Healthy,
WorkerState::Degraded,
WorkerState::Unhealthy,
];
pub fn as_str(self) -> &'static str {
match self {
WorkerState::Healthy => "healthy",
WorkerState::Degraded => "degraded",
WorkerState::Unhealthy => "unhealthy",
}
}
}
/// Emit the worker health state as a state-set gauge: one series per state
/// labeled `state`, with the current state at 1 and all others at 0 (same
/// pattern as `kube_pod_status_phase`). Operators can alert on a state by
/// name without decoding magic numbers.
fn set_state_gauge(worker_url: &str, current: WorkerState) {
for state in WorkerState::ALL {
gauge!(
"ingestion_consumer_worker_health_state",
"worker" => worker_url.to_string(),
"state" => state.as_str(),
)
.set(if state == current { 1.0 } else { 0.0 });
}
}
/// Clear the state-set gauge for a worker that has left the pool, so a departed
/// worker doesn't linger at 1 in dashboards.
fn clear_state_gauge(worker_url: &str) {
for state in WorkerState::ALL {
gauge!(
"ingestion_consumer_worker_health_state",
"worker" => worker_url.to_string(),
"state" => state.as_str(),
)
.set(0.0);
}
}
/// Per-worker mutable health state.
struct WorkerHealth {
state: WorkerState,
state_entered_at: Instant,
/// When the worker entered Unhealthy. None if the worker is not currently Unhealthy.
unhealthy_since: Option<Instant>,
consecutive_probe_failures: u32,
/// Rolling window of passive send outcomes: (timestamp, is_error).
/// Always appended in chronological order so the head is always the oldest entry.
passive_window: VecDeque<(Instant, bool)>,
/// The worker is draining: it left the pool (e.g. a deploy) but may still be
/// finishing in-flight batches. It receives no new work and its `/_ready`
/// probe failures are ignored (readiness is expected to be down), but send
/// failures still escalate it (a crash mid-drain). It is reaped once
/// `drain_deadline` passes — set to "now" when its in-flight reaches zero.
drain_deadline: Option<Instant>,
}
impl WorkerHealth {
fn new() -> Self {
Self {
state: WorkerState::Healthy,
state_entered_at: Instant::now(),
unhealthy_since: None,
consecutive_probe_failures: 0,
passive_window: VecDeque::new(),
drain_deadline: None,
}
}
fn is_draining(&self) -> bool {
self.drain_deadline.is_some()
}
/// Whether this worker is currently eligible for routing: Healthy or
/// Degraded, and not draining. The single definition behind both
/// [`WorkerRegistry::healthy_workers`] and the debug snapshot.
fn is_routable(&self) -> bool {
!self.is_draining() && matches!(self.state, WorkerState::Healthy | WorkerState::Degraded)
}
/// Attempt a state transition. Returns true if the transition happened.
///
/// Transitions to the same state are ignored. Transitions within
/// `min_state_duration` of the last transition are blocked to prevent
/// flapping.
fn try_transition(
&mut self,
new_state: WorkerState,
min_state_duration: Duration,
worker_url: &str,
recorder: &Option<Arc<DebugRecorder>>,
) -> bool {
if self.state == new_state {
return false;
}
if self.state_entered_at.elapsed() < min_state_duration {
return false;
}
let old_state = self.state;
self.state = new_state;
self.state_entered_at = Instant::now();
// Reset so probe failure count is always scoped to the current state
// epoch. Without this, a passive-driven transition (e.g. Healthy →
// Degraded) carries the stale count into the new state, effectively
// lowering the probe_failure_threshold by however many failures had
// already accumulated.
self.consecutive_probe_failures = 0;
if new_state == WorkerState::Unhealthy && self.unhealthy_since.is_none() {
self.unhealthy_since = Some(Instant::now());
} else if new_state != WorkerState::Unhealthy {
self.unhealthy_since = None;
}
info!(
worker = %worker_url,
from = old_state.as_str(),
to = new_state.as_str(),
"Worker state transition"
);
counter!(
"ingestion_consumer_worker_state_transitions_total",
"worker" => worker_url.to_string(),
"from" => old_state.as_str(),
"to" => new_state.as_str(),
)
.increment(1);
set_state_gauge(worker_url, new_state);
record_if(recorder, || DebugEventKind::WorkerStateChanged {
worker: worker_url.to_string(),
from: old_state.as_str(),
to: new_state.as_str(),
});
true
}
fn is_dead(&self, dead_declaration: Duration) -> bool {
self.state == WorkerState::Unhealthy
&& self
.unhealthy_since
.is_some_and(|t| t.elapsed() >= dead_declaration)
}
/// Prune passive window entries older than `window` and return
/// `(error_rate, sample_count)` over the remaining entries.
fn passive_error_rate(&mut self, window: Duration) -> (f64, usize) {
let now = Instant::now();
while let Some((t, _)) = self.passive_window.front() {
if now.saturating_duration_since(*t) > window {
self.passive_window.pop_front();
} else {
break;
}
}
let total = self.passive_window.len();
if total == 0 {
return (0.0, 0);
}
let errors = self
.passive_window
.iter()
.filter(|(_, is_err)| *is_err)
.count();
(errors as f64 / total as f64, total)
}
}
pub struct WorkerRegistryConfig {
pub probe_interval: Duration,
pub dead_declaration: Duration,
pub passive_window: Duration,
pub passive_error_threshold: f64,
pub passive_min_samples: usize,
pub degraded_hold: Duration,
pub min_state_duration: Duration,
pub probe_failure_threshold: u32,
pub drain_timeout: Duration,
}
impl From<&Config> for WorkerRegistryConfig {
fn from(c: &Config) -> Self {
Self {
probe_interval: Duration::from_millis(c.worker_probe_interval_ms),
dead_declaration: Duration::from_millis(c.worker_dead_declaration_ms),
passive_window: Duration::from_millis(c.worker_passive_window_ms),
passive_error_threshold: c.worker_passive_error_threshold,
passive_min_samples: c.worker_passive_min_samples,
degraded_hold: Duration::from_millis(c.worker_degraded_hold_ms),
min_state_duration: Duration::from_millis(c.worker_min_state_duration_ms),
probe_failure_threshold: c.worker_probe_failure_threshold,
drain_timeout: Duration::from_millis(c.worker_drain_timeout_ms),
}
}
}
/// Tracks the health state of each Node.js worker, keyed by worker URL so the
/// set can change at runtime (workers joining/leaving the pool). Uses two
/// independent signals:
///
/// - **Active probe**: GET `/_ready` every `probe_interval`. Two consecutive
/// failures move the worker to `Unhealthy`; one success starts the recovery
/// to `Degraded` (held for `degraded_hold`), then `Healthy`.
/// - **Passive signal**: rolling window of `send_batch` outcomes. If error rate
/// exceeds `passive_error_threshold` with at least `passive_min_samples`
/// samples, the worker moves one step toward `Unhealthy`. Only probe success
/// drives recovery.
///
/// All transitions respect a `min_state_duration` cooldown to prevent flapping.
///
/// A worker that has been `Unhealthy` for at least `dead_declaration` is
/// considered **dead**. A worker removed from the pool entirely is also
/// reported dead.
pub struct WorkerRegistry {
workers: DashMap<WorkerId, WorkerHealth>,
config: WorkerRegistryConfig,
client: reqwest::Client,
/// Debug event recorder; `None` unless `DEBUG_API_ENABLED`.
debug_recorder: Option<Arc<DebugRecorder>>,
}
impl WorkerRegistry {
pub fn new(worker_urls: &[String], config: WorkerRegistryConfig) -> Self {
// Probe timeout is half the probe interval so probes don't overlap.
let probe_timeout = config.probe_interval / 2;
let client = reqwest::Client::builder()
.timeout(probe_timeout)
.build()
.expect("failed to create probe HTTP client");
let workers = DashMap::new();
for url in worker_urls {
workers.insert(WorkerId::from(url.as_str()), WorkerHealth::new());
// Emit the initial state so the gauge exists from startup rather
// than only after the first transition.
set_state_gauge(url, WorkerState::Healthy);
}
Self {
workers,
config,
client,
debug_recorder: None,
}
}
/// Inject the debug UI recorder. Call before the registry is shared.
pub fn set_debug_recorder(&mut self, recorder: Arc<DebugRecorder>) {
self.debug_recorder = Some(recorder);
}
/// Per-worker health snapshot for the debug UI. Prunes each worker's
/// passive window as a side effect (same as the metrics path).
pub fn health_snapshots(&self) -> Vec<WorkerHealthSnapshot> {
let passive_window = self.config.passive_window;
self.workers
.iter_mut()
.map(|mut entry| {
let url = entry.key().to_string();
let health = entry.value_mut();
let (passive_error_rate, passive_samples) =
health.passive_error_rate(passive_window);
WorkerHealthSnapshot {
url,
state: health.state.as_str().to_string(),
draining: health.is_draining(),
routable: health.is_routable(),
consecutive_probe_failures: health.consecutive_probe_failures,
passive_error_rate,
passive_samples,
}
})
.collect()
}
pub fn worker_count(&self) -> usize {
self.workers.len()
}
/// Snapshot of all current worker ids.
pub fn workers(&self) -> Vec<WorkerId> {
self.workers.iter().map(|e| e.key().clone()).collect()
}
/// Snapshot of the workers that discovery lists, so not draining, in any
/// health state. Every consumer of the same Service sees the same set.
pub fn members(&self) -> Vec<WorkerId> {
self.workers
.iter()
.filter(|e| !e.value().is_draining())
.map(|e| e.key().clone())
.collect()
}
/// Snapshot of worker ids currently eligible for routing: Healthy or
/// Degraded, and not draining. Returned in arbitrary order.
pub fn healthy_workers(&self) -> Vec<WorkerId> {
self.workers
.iter()
.filter(|e| e.value().is_routable())
.map(|e| e.key().clone())
.collect()
}
/// Current state of a worker. An unknown worker (already removed from the
/// pool) reports `Unhealthy` so callers never route to it.
pub fn state(&self, worker: &str) -> WorkerState {
self.workers
.get(worker)
.map(|h| h.state)
.unwrap_or(WorkerState::Unhealthy)
}
/// Whether a worker is dead — Unhealthy for at least `dead_declaration`, or
/// no longer in the pool at all.
pub fn is_dead(&self, worker: &str) -> bool {
self.workers
.get(worker)
.map(|h| h.is_dead(self.config.dead_declaration))
.unwrap_or(true)
}
/// Add a worker to the pool. If it is already present and draining (it left
/// and rejoined, e.g. a flapping EndpointSlice), clear the draining mark so
/// it routes again; otherwise an existing worker keeps its current health.
pub fn add_worker(&self, worker: WorkerId) {
// `entry` makes the check-and-insert atomic — a plain `contains_key` +
// `insert` could let a concurrent caller overwrite a fresh `WorkerHealth`
// and reset health state.
match self.workers.entry(worker.clone()) {
Entry::Occupied(mut slot) => {
// Already present. If it was draining (it left and rejoined, e.g.
// a flapping EndpointSlice), clear the draining mark so it routes
// again; otherwise it keeps its current health.
if slot.get_mut().drain_deadline.take().is_some() {
info!(worker = %worker, "Draining worker rejoined the pool");
}
}
Entry::Vacant(slot) => {
set_state_gauge(&worker, WorkerState::Healthy);
info!(worker = %worker, "Worker added to pool");
slot.insert(WorkerHealth::new());
counter!("ingestion_consumer_worker_membership_total", "action" => "add")
.increment(1);
record_if(&self.debug_recorder, || DebugEventKind::WorkerMembership {
worker: worker.to_string(),
action: "add",
});
}
}
}
/// Mark a worker as draining: it stops receiving new work (excluded from
/// `healthy_workers`) but is not treated as dead, so its already-sent
/// in-flight batches finish and ACK normally. Probe failures are ignored
/// while draining (its readiness is expected to be down), but send failures
/// still escalate it (a crash mid-drain). The reaper removes it once
/// `complete_drain` fires (in-flight reached zero) or `drain_timeout` elapses.
pub fn start_draining(&self, worker: &str) {
if let Some(mut health) = self.workers.get_mut(worker) {
if health.is_draining() {
return;
}
health.drain_deadline = Some(Instant::now() + self.config.drain_timeout);
info!(worker = %worker, "Worker draining (no new work; finishing in-flight)");
counter!("ingestion_consumer_worker_membership_total", "action" => "drain")
.increment(1);
record_if(&self.debug_recorder, || DebugEventKind::WorkerMembership {
worker: worker.to_string(),
action: "drain",
});
}
}
/// Mark a draining worker ready to reap now — called once its in-flight count
/// reaches zero, so it's removed promptly instead of waiting for the timeout.
pub fn complete_drain(&self, worker: &str) {
if let Some(mut health) = self.workers.get_mut(worker) {
if health.is_draining() {
health.drain_deadline = Some(Instant::now());
}
}
}
pub fn is_draining(&self, worker: &str) -> bool {
self.workers
.get(worker)
.map(|h| h.is_draining())
.unwrap_or(false)
}
/// Draining workers whose reap deadline has passed (in-flight drained, or the
/// timeout elapsed). The caller removes them from the registry and transport.
pub fn reapable_workers(&self) -> Vec<WorkerId> {
let now = Instant::now();
self.workers
.iter()
.filter(|e| e.value().drain_deadline.is_some_and(|d| now >= d))
.map(|e| e.key().clone())
.collect()
}
/// All workers currently draining (regardless of deadline). The reaper uses
/// this to complete the drain of a worker that left the pool while idle.
pub fn draining_workers(&self) -> Vec<WorkerId> {
self.workers
.iter()
.filter(|e| e.value().is_draining())
.map(|e| e.key().clone())
.collect()
}
/// Remove a worker from the pool (e.g. it left the EndpointSlice).
pub fn remove_worker(&self, worker: &str) {
if self.workers.remove(worker).is_some() {
clear_state_gauge(worker);
info!(worker = %worker, "Worker removed from pool");
counter!("ingestion_consumer_worker_membership_total", "action" => "remove")
.increment(1);
record_if(&self.debug_recorder, || DebugEventKind::WorkerMembership {
worker: worker.to_string(),
action: "remove",
});
}
}
/// Record the outcome of a `send_batch` call for the passive signal.
pub fn record_outcome(&self, worker: &str, is_error: bool) {
let Some(mut health) = self.workers.get_mut(worker) else {
return;
};
health.passive_window.push_back((Instant::now(), is_error));
if health.passive_window.len() > PASSIVE_WINDOW_MAX_ENTRIES {
health.passive_window.pop_front();
}
let (error_rate, sample_count) = health.passive_error_rate(self.config.passive_window);
gauge!(
"ingestion_consumer_worker_passive_error_rate",
"worker" => worker.to_string(),
)
.set(error_rate);
if sample_count >= self.config.passive_min_samples
&& error_rate > self.config.passive_error_threshold
{
let target = match health.state {
WorkerState::Healthy => Some(WorkerState::Degraded),
WorkerState::Degraded => Some(WorkerState::Unhealthy),
WorkerState::Unhealthy => None,
};
if let Some(new_state) = target {
health.try_transition(
new_state,
self.config.min_state_duration,
worker,
&self.debug_recorder,
);
}
}
}
/// Spawn a single background task that probes every current worker each
/// `probe_interval`. Runs until `token` is cancelled. The worker set is
/// re-read each tick, so workers added/removed at runtime are picked up.
pub fn start_probing(self: Arc<Self>, token: CancellationToken) {
tokio::spawn(async move {
loop {
tokio::select! {
_ = token.cancelled() => break,
_ = tokio::time::sleep(self.config.probe_interval) => {}
}
let workers = self.workers();
let probes = workers.into_iter().map(|worker| {
let registry = Arc::clone(&self);
async move { registry.probe_once(&worker).await }
});
futures::future::join_all(probes).await;
}
});
}
async fn probe_once(&self, worker: &str) {
let start = Instant::now();
let result = self.client.get(format!("{worker}/_ready")).send().await;
let elapsed = start.elapsed();
let is_success = result.map(|r| r.status().is_success()).unwrap_or(false);
histogram!(
"ingestion_consumer_worker_probe_duration_seconds",
"worker" => worker.to_string(),
"outcome" => if is_success { "success" } else { "failure" },
)
.record(elapsed.as_secs_f64());
// The worker may have been removed between the snapshot and now.
let Some(mut health) = self.workers.get_mut(worker) else {
return;
};
// A draining worker's `/_ready` is expected to be down (it flipped
// readiness to leave the pool), so ignore probe results — otherwise the
// probe would march it to dead and evict its in-flight work. A crash
// mid-drain is still caught by the passive send-failure signal.
if health.is_draining() {
return;
}
if is_success {
health.consecutive_probe_failures = 0;
let new_state = match health.state {
WorkerState::Unhealthy => Some(WorkerState::Degraded),
WorkerState::Degraded => {
if health.state_entered_at.elapsed() >= self.config.degraded_hold {
Some(WorkerState::Healthy)
} else {
None
}
}
WorkerState::Healthy => None,
};
if let Some(s) = new_state {
health.try_transition(
s,
self.config.min_state_duration,
worker,
&self.debug_recorder,
);
}
} else {
health.consecutive_probe_failures += 1;
warn!(
worker = %worker,
consecutive_failures = health.consecutive_probe_failures,
"Worker probe failed"
);
if health.consecutive_probe_failures >= self.config.probe_failure_threshold {
health.try_transition(
WorkerState::Unhealthy,
self.config.min_state_duration,
worker,
&self.debug_recorder,
);
}
}
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use super::*;
const W1: &str = "http://worker:9001";
const W2: &str = "http://worker:9002";
const W3: &str = "http://worker:9003";
fn no_cooldown_config() -> WorkerRegistryConfig {
WorkerRegistryConfig {
probe_interval: Duration::from_millis(50),
dead_declaration: Duration::from_millis(100),
passive_window: Duration::from_millis(500),
passive_error_threshold: 0.5,
passive_min_samples: 3,
degraded_hold: Duration::from_millis(50),
min_state_duration: Duration::ZERO,
probe_failure_threshold: 2,
drain_timeout: Duration::from_secs(5),
}
}
fn one_worker() -> Vec<String> {
vec![W1.to_string()]
}
fn three_workers() -> Vec<String> {
vec![W1.to_string(), W2.to_string(), W3.to_string()]
}
// --- Passive signal transitions ---
#[test]
fn test_passive_healthy_to_degraded() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
// 3 errors = 100% error rate, 3 samples ≥ min_samples=3, rate > threshold=0.5
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Degraded);
}
#[test]
fn test_passive_degraded_to_unhealthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
for _ in 0..3 {
registry.record_outcome(W1, true);
}
assert_eq!(registry.state(W1), WorkerState::Degraded);
// One more error pushes Degraded → Unhealthy (still above threshold)
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Unhealthy);
}
#[test]
fn test_passive_below_threshold_stays_healthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
// 1 error out of 3 = 33% < threshold=50%
registry.record_outcome(W1, true);
registry.record_outcome(W1, false);
registry.record_outcome(W1, false);
assert_eq!(registry.state(W1), WorkerState::Healthy);
}
#[test]
fn test_passive_below_min_samples_stays_healthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
// 2 errors but only 2 samples < min_samples=3
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Healthy);
}
#[test]
fn test_passive_unhealthy_stays_unhealthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
for _ in 0..4 {
registry.record_outcome(W1, true);
}
assert_eq!(registry.state(W1), WorkerState::Unhealthy);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Unhealthy);
}
// --- Cooldown ---
#[test]
fn test_cooldown_blocks_transition() {
let config = WorkerRegistryConfig {
min_state_duration: Duration::from_secs(100),
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&one_worker(), config);
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Healthy);
}
// --- Probe-driven transitions (via direct WorkerHealth manipulation) ---
#[test]
fn test_probe_failures_to_unhealthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
let mut health = registry.workers.get_mut(W1).unwrap();
health.consecutive_probe_failures = 2;
let transitioned = health.try_transition(WorkerState::Unhealthy, Duration::ZERO, W1, &None);
assert!(transitioned);
assert_eq!(health.state, WorkerState::Unhealthy);
assert!(health.unhealthy_since.is_some());
}
#[test]
fn test_probe_success_unhealthy_to_degraded() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
let mut health = registry.workers.get_mut(W1).unwrap();
health.try_transition(WorkerState::Unhealthy, Duration::ZERO, W1, &None);
assert_eq!(health.state, WorkerState::Unhealthy);
let transitioned = health.try_transition(WorkerState::Degraded, Duration::ZERO, W1, &None);
assert!(transitioned);
assert_eq!(health.state, WorkerState::Degraded);
assert!(health.unhealthy_since.is_none());
}
#[test]
fn test_probe_failure_threshold_not_reached() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
let health = registry.workers.get(W1).unwrap();
assert_eq!(health.state, WorkerState::Healthy);
}
// --- Dead declaration ---
#[tokio::test]
async fn test_dead_declaration_fires_after_duration() {
let config = WorkerRegistryConfig {
dead_declaration: Duration::from_millis(30),
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&one_worker(), config);
for _ in 0..4 {
registry.record_outcome(W1, true);
}
assert_eq!(registry.state(W1), WorkerState::Unhealthy);
assert!(!registry.is_dead(W1));
tokio::time::sleep(Duration::from_millis(40)).await;
assert!(registry.is_dead(W1));
}
#[test]
fn test_healthy_worker_is_not_dead() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
assert!(!registry.is_dead(W1));
}
// --- Membership ---
#[test]
fn test_unknown_worker_is_dead_and_unhealthy() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
assert!(registry.is_dead("http://worker:9999"));
assert_eq!(registry.state("http://worker:9999"), WorkerState::Unhealthy);
}
#[test]
fn test_add_worker_makes_it_healthy_and_routable() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
assert_eq!(registry.worker_count(), 1);
registry.add_worker(WorkerId::from(W2));
assert_eq!(registry.worker_count(), 2);
assert_eq!(registry.state(W2), WorkerState::Healthy);
assert!(registry.healthy_workers().iter().any(|w| w.as_ref() == W2));
}
#[test]
fn test_add_worker_is_idempotent_and_preserves_state() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
// Drive W1 to Degraded, then re-add it — must not reset to Healthy.
for _ in 0..3 {
registry.record_outcome(W1, true);
}
assert_eq!(registry.state(W1), WorkerState::Degraded);
registry.add_worker(WorkerId::from(W1));
assert_eq!(registry.worker_count(), 1);
assert_eq!(registry.state(W1), WorkerState::Degraded);
}
#[test]
fn test_remove_worker_drops_it_from_pool() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
registry.remove_worker(W2);
assert_eq!(registry.worker_count(), 2);
assert!(registry.is_dead(W2));
assert!(!registry.healthy_workers().iter().any(|w| w.as_ref() == W2));
}
#[test]
fn test_healthy_workers_excludes_unhealthy() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
for _ in 0..4 {
registry.record_outcome(W2, true);
}
assert_eq!(registry.state(W2), WorkerState::Unhealthy);
let healthy = registry.healthy_workers();
assert_eq!(healthy.len(), 2);
assert!(!healthy.iter().any(|w| w.as_ref() == W2));
}
#[test]
fn test_members_keep_unhealthy_workers_and_drop_draining_ones() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
for _ in 0..4 {
registry.record_outcome(W2, true);
}
registry.start_draining(W3);
let mut members: Vec<String> = registry.members().iter().map(|w| w.to_string()).collect();
members.sort();
assert_eq!(members, vec![W1.to_string(), W2.to_string()]);
}
// --- Draining lifecycle ---
#[test]
fn test_draining_excludes_from_routing_but_not_dead() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
registry.start_draining(W2);
assert!(registry.is_draining(W2));
assert!(
!registry.is_dead(W2),
"a draining worker is alive, not dead"
);
assert_eq!(
registry.state(W2),
WorkerState::Healthy,
"draining doesn't change the health state"
);
assert!(
!registry.healthy_workers().iter().any(|w| w.as_ref() == W2),
"a draining worker is not routable"
);
assert_eq!(registry.worker_count(), 3, "still in the pool until reaped");
}
#[test]
fn test_complete_drain_makes_reapable() {
// Long timeout, so only complete_drain (in-flight reached zero) reaps it.
let config = WorkerRegistryConfig {
drain_timeout: Duration::from_secs(3600),
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&three_workers(), config);
registry.start_draining(W2);
assert!(
registry.reapable_workers().is_empty(),
"not reapable until drained or timed out"
);
registry.complete_drain(W2);
assert_eq!(registry.reapable_workers(), vec![WorkerId::from(W2)]);
}
#[tokio::test]
async fn test_drain_timeout_makes_reapable() {
let config = WorkerRegistryConfig {
drain_timeout: Duration::from_millis(20),
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&one_worker(), config);
registry.start_draining(W1);
assert!(registry.reapable_workers().is_empty());
tokio::time::sleep(Duration::from_millis(30)).await;
assert_eq!(registry.reapable_workers(), vec![WorkerId::from(W1)]);
}
#[test]
fn test_add_worker_rejoin_clears_draining() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
registry.start_draining(W1);
assert!(registry.is_draining(W1));
registry.add_worker(WorkerId::from(W1));
assert!(!registry.is_draining(W1), "rejoining clears draining");
assert!(registry.healthy_workers().iter().any(|w| w.as_ref() == W1));
}
#[test]
fn test_start_draining_unknown_worker_is_noop() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
registry.start_draining("http://worker:9999");
assert!(!registry.is_draining("http://worker:9999"));
assert!(registry.reapable_workers().is_empty());
}
// --- Passive window pruning ---
#[tokio::test]
async fn test_passive_window_prunes_old_entries() {
let config = WorkerRegistryConfig {
passive_window: Duration::from_millis(30),
passive_min_samples: 2,
passive_error_threshold: 0.5,
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&one_worker(), config);
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Degraded);
tokio::time::sleep(Duration::from_millis(40)).await;
registry.record_outcome(W1, false);
registry.record_outcome(W1, false);
let (rate, count) = {
let mut health = registry.workers.get_mut(W1).unwrap();
health.passive_error_rate(Duration::from_millis(30))
};
assert_eq!(count, 2);
assert_eq!(rate, 0.0);
}
// --- Multiple workers ---
#[test]
fn test_worker_count() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
assert_eq!(registry.worker_count(), 3);
}
#[test]
fn test_states_are_independent_per_worker() {
let registry = WorkerRegistry::new(&three_workers(), no_cooldown_config());
for _ in 0..4 {
registry.record_outcome(W2, true);
}
assert_eq!(registry.state(W1), WorkerState::Healthy);
assert_eq!(registry.state(W2), WorkerState::Unhealthy);
assert_eq!(registry.state(W3), WorkerState::Healthy);
}
#[tokio::test]
async fn test_is_dead_is_per_worker() {
let config = WorkerRegistryConfig {
dead_declaration: Duration::from_millis(30),
..no_cooldown_config()
};
let registry = WorkerRegistry::new(&three_workers(), config);
for _ in 0..4 {
registry.record_outcome(W3, true);
}
tokio::time::sleep(Duration::from_millis(40)).await;
assert!(!registry.is_dead(W1));
assert!(!registry.is_dead(W2));
assert!(registry.is_dead(W3));
}
// --- consecutive_probe_failures reset on transition ---
#[test]
fn test_probe_failure_count_reset_on_passive_transition() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
registry
.workers
.get_mut(W1)
.unwrap()
.consecutive_probe_failures = 1;
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
registry.record_outcome(W1, true);
assert_eq!(registry.state(W1), WorkerState::Degraded);
assert_eq!(
registry.workers.get(W1).unwrap().consecutive_probe_failures,
0,
"try_transition must reset consecutive_probe_failures so the Degraded epoch needs a full probe_failure_threshold failures to reach Unhealthy"
);
}
// --- passive_window size cap ---
#[test]
fn test_passive_window_capped_at_max_entries() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
for _ in 0..PASSIVE_WINDOW_MAX_ENTRIES + 10 {
registry.record_outcome(W1, false);
}
assert!(
registry.workers.get(W1).unwrap().passive_window.len() <= PASSIVE_WINDOW_MAX_ENTRIES,
"passive_window must not exceed PASSIVE_WINDOW_MAX_ENTRIES"
);
}
// --- Same-state no-op ---
#[test]
fn test_transition_to_same_state_is_noop() {
let registry = WorkerRegistry::new(&one_worker(), no_cooldown_config());
let mut health = registry.workers.get_mut(W1).unwrap();
let transitioned = health.try_transition(WorkerState::Healthy, Duration::ZERO, W1, &None);
assert!(!transitioned);
assert_eq!(health.state, WorkerState::Healthy);
}
}