:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use arc_swap::ArcSwap;
use dashmap::DashSet;
use siphasher::sip128::SipHasher13;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use common_redis::Client;
use moka::sync::Cache;
use tokio::sync::mpsc;
use tracing::{error, info, warn};
use crate::custom_key_source::CustomKeyThresholdSource;
/// Resolver for custom-key thresholds. Given a lookup key and a snapshot of the
/// current custom-key map, returns the threshold to apply (or `None` if the key
/// is not subject to a custom limit).
///
/// The default resolver (when `custom_key_resolver` is `None`) is an exact map
/// lookup. Callers can inject a closure to implement richer policies (e.g. a
/// hierarchical `token:distinct_id` -> `token` fallback) without leaking their
/// key structure into this crate. The closure receives the authoritative map by
/// reference so it always resolves against the latest swapped-in thresholds.
pub type CustomKeyResolver = Arc<dyn Fn(&str, &HashMap<String, u64>) -> Option<u64> + Send + Sync>;
const GLOBAL_RATE_LIMITER_EVAL_COUNTER: &str = "global_rate_limiter_eval_counts_total";
const GLOBAL_RATE_LIMITER_CACHE_COUNTER: &str = "global_rate_limiter_cache_counts_total";
const GLOBAL_RATE_LIMITER_RECORDS_COUNTER: &str = "global_rate_limiter_records_total";
const GLOBAL_RATE_LIMITER_ERROR_COUNTER: &str = "global_rate_limiter_error_total";
const GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM: &str = "global_rate_limiter_pipeline_ms";
const GLOBAL_RATE_LIMITER_TICK_HISTOGRAM: &str = "global_rate_limiter_tick_ms";
const GLOBAL_RATE_LIMITER_PIPELINE_SIZE_HISTOGRAM: &str = "global_rate_limiter_pipeline_size";
const GLOBAL_RATE_LIMITER_PENDING_SYNC_SIZE_GAUGE: &str = "global_rate_limiter_pending_sync_size";
const GLOBAL_RATE_LIMITER_WINDOW_GAUGE: &str = "global_rate_limiter_window_seconds";
const GLOBAL_RATE_LIMITER_SYNC_TIER_GAUGE: &str = "global_rate_limiter_sync_tier_gauge";
const GLOBAL_RATE_LIMITER_TIER_TRANSITIONS_COUNTER: &str =
"global_rate_limiter_tier_transitions_total";
const GLOBAL_RATE_LIMITER_ESTIMATE_DRIFT_HISTOGRAM: &str = "global_rate_limiter_estimate_drift";
const GLOBAL_RATE_LIMITER_SYNC_STALENESS_HISTOGRAM: &str = "global_rate_limiter_sync_staleness_ms";
const GLOBAL_RATE_LIMITER_CACHE_SIZE_GAUGE: &str = "global_rate_limiter_cache_size";
const GLOBAL_RATE_LIMITER_EVICTION_COUNTER: &str = "global_rate_limiter_eviction_total";
/// Keys still queued for sync after a tick took its bounded slice.
const GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE: &str = "global_rate_limiter_sync_deferred_size";
/// (key, epoch) write entries still batched after a tick took its bounded slice.
const GLOBAL_RATE_LIMITER_WRITE_DEFERRED_GAUGE: &str = "global_rate_limiter_write_deferred_size";
/// Updates waiting in the channel when a tick starts.
const GLOBAL_RATE_LIMITER_CHANNEL_DEPTH_GAUGE: &str = "global_rate_limiter_update_channel_depth";
/// Syncs not queued because the key's level is below `min_sync_floor`.
const GLOBAL_RATE_LIMITER_SYNC_SKIPPED_COUNTER: &str = "global_rate_limiter_sync_skipped_total";
/// Redis commands issued per tick, after chunking.
const GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM: &str = "global_rate_limiter_commands_per_tick";
/// Seconds since reads to one Redis instance began failing; zero while they succeed.
const GLOBAL_RATE_LIMITER_READ_FAILING_SECONDS_GAUGE: &str =
"global_rate_limiter_read_failing_seconds";
/// Limits withheld because the key's Redis instance was in a read outage. A
/// separate counter, since `cache_counts_total` has one result per evaluation.
const GLOBAL_RATE_LIMITER_OUTAGE_LIMIT_SKIPPED_COUNTER: &str =
"global_rate_limiter_outage_limit_skipped_total";
/// Number of custom-key thresholds applied at the last successful refresh.
const CUSTOM_THRESHOLDS_LOADED_GAUGE: &str = "global_rate_limiter_custom_thresholds_loaded";
/// Unix timestamp of the last successful custom-key threshold refresh.
const CUSTOM_THRESHOLDS_LAST_REFRESH_GAUGE: &str =
"global_rate_limiter_custom_thresholds_last_refresh_timestamp";
/// Full-cache scan cadence (ticks) for the per-tier distribution gauges. The
/// distribution moves slowly and prod metrics dedup to 60s, so a periodic scan
/// keeps these gauges fresh enough without scanning the cache every tick.
const TIER_SCAN_INTERVAL_TICKS: u64 = 30;
/// Batch receives so Tokio's 128-receive poll budget cannot throttle the drain on a starved pod.
const UPDATE_DRAIN_BATCH: usize = 256;
/// Tier label order, indexed by `PressureTier::index()`.
const TIER_LABELS: [&str; 4] = ["idle", "low", "normal", "hot"];
/// Pressure tiers for adaptive sync scheduling
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PressureTier {
/// < 10% capacity: no cadence of its own. Above `min_sync_floor` it syncs on the Low cadence
Idle,
/// 10-50% capacity: sync at 4x sync_interval
Low,
/// 50-80% capacity: sync at 1x sync_interval
Normal,
/// >= 80% capacity: sync at sync_interval / 2
Hot,
}
impl PressureTier {
pub fn from_pressure(pressure: f64) -> Self {
if pressure < 0.1 {
Self::Idle
} else if pressure < 0.5 {
Self::Low
} else if pressure < 0.8 {
Self::Normal
} else {
Self::Hot
}
}
pub fn as_str(&self) -> &'static str {
match self {
Self::Idle => "idle",
Self::Low => "low",
Self::Normal => "normal",
Self::Hot => "hot",
}
}
/// Index into the per-tier scan tally array (see `TIER_LABELS`).
pub fn index(&self) -> usize {
match self {
Self::Idle => 0,
Self::Low => 1,
Self::Normal => 2,
Self::Hot => 3,
}
}
}
/// Compute the effective sync interval for a given pressure tier
pub fn tier_sync_interval(pressure: f64, base_sync_interval: Duration) -> Option<Duration> {
match PressureTier::from_pressure(pressure) {
PressureTier::Idle => None, // no cadence of its own; the caller uses the Low cadence
PressureTier::Low => Some(base_sync_interval.mul_f64(4.0)),
PressureTier::Normal => Some(base_sync_interval),
PressureTier::Hot => Some(base_sync_interval.div_f64(2.0)),
}
}
/// Trait for global rate limiting
#[async_trait]
pub trait GlobalRateLimiter: Send + Sync {
/// Check if a key is rate limited, recording the count for this request.
///
/// - Consult the local cache with leaky bucket decay
/// - Enqueue an update to the key's count for async batch submission
/// - Push to pending_sync if sync interval exceeded
/// - Allow the request if the local cache has no entry for the key yet
///
/// Returns `EvalResult` indicating whether the request is allowed or limited
async fn check_limit(
&self,
key: &str,
count: u64,
timestamp: Option<DateTime<Utc>>,
) -> EvalResult;
/// Check if a "custom key" is rate limited, recording the count for this request.
/// The operation is the same as `check_limit`, other than how the key and threshold
/// are determined:
///
/// - Custom keys are defined in the custom_keys map, associated with an override value
/// - If the key is present in the map, the override threshold value is applied
/// - If the key is not present in the map, it is not subject to rate limiting
///
/// Returns `EvalResult` indicating whether the request is allowed, limited, or not applicable
async fn check_custom_limit(
&self,
key: &str,
count: u64,
timestamp: Option<DateTime<Utc>>,
) -> EvalResult;
/// Returns true if the key is registered in the custom_keys map
fn is_custom_key(&self, key: &str) -> bool;
/// Close the update channel and flush remaining update records to global cache
fn shutdown(&mut self);
}
/// Configuration for the global rate limiter
#[derive(Clone)]
pub struct GlobalRateLimiterConfig {
/// Maximum count allowed per window for a given key (default for keys not in custom_keys)
pub global_threshold: u64,
/// Sliding window size (e.g., 60 seconds) - defines the 2-epoch counter size
pub window_interval: Duration,
/// Base re-sync cadence (default 15s). The pressure tier scales it: Hot is
/// half, Normal is 1x, Low and Idle are 4x
pub sync_interval: Duration,
/// Background task cadence for pipeline reads + writes (default 1s)
pub tick_interval: Duration,
/// Redis key prefix (not including final separator)
pub redis_key_prefix: String,
/// Epoch keys expire at an absolute deadline, so only the part of this above
/// 2 * window_interval is used, as clock-skew grace
pub global_cache_ttl: Duration,
/// Time since an entry was last written before moka drops it. Every request
/// and every read rewrites the entry, so this restarts each time
pub local_cache_ttl: Duration,
/// Evict entries not accessed within this window. Hot keys are constantly
/// re-inserted so they never idle-expire; cold keys reclaim slots faster
/// than waiting for the full TTL.
pub local_cache_idle_timeout: Duration,
/// Timeout for global cache read operations
pub global_read_timeout: Duration,
/// Timeout for global cache write operations
pub global_write_timeout: Duration,
/// Maximum entries in the local cache. Moka's TinyLFU policy can refuse a
/// new key at the cap
pub local_cache_max_entries: u64,
/// Capacity of the mpsc channel for async global cache updates
pub channel_capacity: usize,
/// Minimum local level before a key earns a Redis read, which keeps one-shot keys off
/// the pipeline; `0` reads every key. Keep it well under `global_threshold / node_count`,
/// or a key at its limit spread across the fleet is read more than a window late.
pub min_sync_floor: u64,
/// Wall-clock time reads to an instance must keep failing before its keys stop
/// limiting on this node's own counts. `None` uses `window_interval`; `Some(ZERO)` disables.
pub max_read_outage: Option<Duration>,
/// Max keys read, and max write entries sent, per tick. The rest wait, so a backlog
/// becomes staleness instead of a tick that overruns its interval.
pub max_sync_keys_per_tick: usize,
/// Maximum Redis keys per individual command. Reads cost two keys per entity
/// (current + previous epoch), so an entity chunk is half this. Bounds how
/// long any single command can take, which is what the per-command timeout
/// is actually budgeting for.
pub max_keys_per_command: usize,
/// How many chunked commands may be in flight at once against one instance.
/// Trades tick wall-clock against instantaneous Redis load.
pub max_concurrent_commands: usize,
/// Max distinct `(key, epoch)` entries buffered for writing; at the cap, updates for new
/// keys drop and are counted. The channel's capacity does not bound this map.
pub max_write_batch_entries: usize,
/// Maximum keys held in the pending-sync set. At the cap, new sync
/// requests are dropped and counted; the key's next request re-queues it
/// once the backlog drains. Mirrors `max_write_batch_entries`: without a
/// cap, keys clearing the sync floor faster than the per-tick drain grow
/// the set without bound.
pub max_pending_sync_entries: usize,
/// Per-key custom limits. Overrides the default limit for specific *more granular* keys.
///
/// Wrapped in `Arc<ArcSwap<_>>` so the map can be atomically replaced at
/// runtime (by the refresh task via `custom_keys.store(...)`) without locking
/// hot-path readers. Every clone of the config shares the same underlying
/// `ArcSwap`, so a swap through any clone (e.g. the copy held by the
/// background task) is visible everywhere.
pub custom_keys: Arc<ArcSwap<HashMap<String, u64>>>,
/// Optional policy for resolving a lookup key to a custom threshold. When
/// `None`, resolution is an exact lookup in `custom_keys`. When set, the
/// closure is consulted with the key and a snapshot of the current map.
pub custom_key_resolver: Option<CustomKeyResolver>,
/// Optional source of dynamically-refreshed custom-key thresholds. When set,
/// `GlobalRateLimiterImpl::new` spawns a background task that fetches the map
/// every `custom_key_refresh_interval` and atomically swaps it into
/// `custom_keys`. When `None`, `custom_keys` is static (e.g. seeded once from
/// config) and no refresh task runs.
pub custom_key_source: Option<Arc<dyn CustomKeyThresholdSource>>,
/// Cadence for the custom-key refresh task. Ignored when `custom_key_source`
/// is `None`.
pub custom_key_refresh_interval: Duration,
/// Tag value applied to all metrics emitted by this limiter instance.
/// Allows distinguishing multiple limiter instances in the same process.
pub metrics_scope: String,
}
impl GlobalRateLimiterConfig {
/// Leak rate: tokens per second that drain from the bucket
pub fn leak_rate(&self) -> f64 {
self.global_threshold as f64 / self.window_interval.as_secs_f64()
}
/// Leak rate for a custom key threshold
pub fn leak_rate_for(&self, threshold: u64) -> f64 {
threshold as f64 / self.window_interval.as_secs_f64()
}
/// Resolve a lookup key to its custom threshold, if any.
///
/// Snapshots the current custom-key map once (lock-free) and applies the
/// configured resolver, falling back to an exact lookup when no resolver is
/// injected. Returns `None` when the key is not subject to a custom limit.
pub fn resolve_custom(&self, key: &str) -> Option<u64> {
let guard = self.custom_keys.load();
let map: &HashMap<String, u64> = &guard;
match &self.custom_key_resolver {
Some(resolver) => resolver(key, map),
None => map.get(key).copied(),
}
}
}
impl Default for GlobalRateLimiterConfig {
fn default() -> Self {
let window_interval = Duration::from_secs(60);
Self {
global_threshold: 1_000_000,
window_interval,
sync_interval: Duration::from_secs(15),
tick_interval: Duration::from_secs(1),
redis_key_prefix: "@posthog/global_rate_limiter".to_string(),
local_cache_ttl: Duration::from_secs(600),
local_cache_idle_timeout: Duration::from_secs(300),
global_cache_ttl: window_interval.mul_f64(2.0),
global_read_timeout: Duration::from_millis(250),
global_write_timeout: Duration::from_millis(250),
local_cache_max_entries: 300_000,
channel_capacity: 1_000_000,
min_sync_floor: 10,
max_read_outage: None,
max_sync_keys_per_tick: 20_000,
max_keys_per_command: 2_000,
max_concurrent_commands: 4,
max_write_batch_entries: 200_000,
max_pending_sync_entries: 200_000,
custom_keys: Arc::new(ArcSwap::from_pointee(HashMap::new())),
custom_key_resolver: None,
custom_key_source: None,
custom_key_refresh_interval: Duration::from_secs(60),
metrics_scope: "default".to_string(),
}
}
}
/// Internal struct for caching rate limit state with leaky bucket decay
#[derive(Clone, Debug)]
pub struct CacheEntry {
/// Weighted count from last Redis sync (decays over time via leak_rate)
pub estimated_count: f64,
/// When we last read from Redis; `None` until the first read lands. A
/// never-synced entry is due for a sync as soon as it clears the floor,
/// where a freshly synced one waits out its tier.
pub synced_at: Option<Instant>,
/// Events counted locally since last sync
pub local_pending: u64,
/// effective_level / threshold at last sync, determines adaptive sync tier
pub pressure: f64,
}
/// Compute the effective level of a cache entry with leaky bucket decay.
///
/// The estimate decays the last-known global count by the leak rate and adds
/// locally observed events. This keeps the estimate conservative (includes all
/// local events) while allowing the global contribution to drain away.
pub fn effective_level(entry: &CacheEntry, leak_rate: f64, now: Instant) -> f64 {
decayed_global(entry, leak_rate, now) + entry.local_pending as f64
}
/// The last fleet count read from Redis, decayed to `now`, excluding this node's
/// unconfirmed events; zero if never read.
pub fn decayed_global(entry: &CacheEntry, leak_rate: f64, now: Instant) -> f64 {
let Some(synced_at) = entry.synced_at else {
return 0.0;
};
let elapsed = now.duration_since(synced_at).as_secs_f64();
let drained = leak_rate * elapsed;
(entry.estimated_count - drained).max(0.0)
}
/// Compute the epoch number from a unix timestamp and window interval.
/// epoch = floor(unix_secs / window_interval_secs)
pub fn epoch_from_timestamp(timestamp: DateTime<Utc>, window_interval: Duration) -> i64 {
let unix = timestamp.timestamp();
let window_secs = window_interval.as_secs() as i64;
unix / window_secs
}
/// Build the Redis key for a given entity key and epoch
pub fn epoch_key(prefix: &str, key: &str, epoch: i64) -> String {
format!("{prefix}:{key}:{epoch}")
}
/// Salt that keeps the expiry offset independent of shard selection.
const EXPIRY_HASH_DOMAIN: &str = "grl/expiry-offset";
/// The absolute unix second at which an epoch key stops being readable, since
/// a read consults only the current and previous epoch.
///
/// TTL configured above the two-window minimum becomes clock-skew grace.
pub fn epoch_expire_at(
key: &str,
epoch: i64,
window_interval: Duration,
global_cache_ttl: Duration,
) -> i64 {
let window_secs = window_interval.as_secs() as i64;
let grace_secs = (global_cache_ttl.as_secs() as i64 - 2 * window_secs).max(0);
// Spread expiry over a window. Every key of one epoch shares a deadline, so
// without this the whole generation dies in one instant and that burst
// blocks Redis long enough to time out concurrent writes. The offset only
// ever adds time, so the key still outlives every read that consults it.
// Salted so this hash cannot correlate with `select_redis_client`, which
// hashes the same key. Sharing it would collapse the spread on any shard
// whose count shares a factor with the window.
let mut hasher = DefaultHasher::new();
EXPIRY_HASH_DOMAIN.hash(&mut hasher);
key.hash(&mut hasher);
let jitter_secs = (hasher.finish() % window_secs.max(1) as u64) as i64;
(epoch + 2) * window_secs + grace_secs + jitter_secs
}
/// Build the current and previous epoch Redis keys for a given entity
pub fn epoch_keys(
prefix: &str,
key: &str,
timestamp: DateTime<Utc>,
window_interval: Duration,
) -> (String, String) {
let epoch = epoch_from_timestamp(timestamp, window_interval);
(
epoch_key(prefix, key, epoch),
epoch_key(prefix, key, epoch - 1),
)
}
/// Compute the sliding window counter estimate from two epoch counts.
///
/// progress = fraction of the way through the current epoch (0.0..1.0)
/// estimated_count = prev_count * (1.0 - progress) + current_count
pub fn weighted_count(
prev_count: u64,
current_count: u64,
timestamp: DateTime<Utc>,
window_interval: Duration,
) -> f64 {
let window_secs = window_interval.as_secs_f64();
let unix = timestamp.timestamp() as f64;
let progress = (unix % window_secs) / window_secs;
prev_count as f64 * (1.0 - progress) + current_count as f64
}
/// Request to update a rate limit counter (queued to background task)
struct UpdateRequest {
key: String,
count: u64,
timestamp: DateTime<Utc>,
}
/// Mode for rate limit checking
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CheckMode {
/// Use the global default limit for all keys
Global,
/// Only check keys present in custom_keys map, using their custom limits
Custom,
}
/// When each instance's current run of failed reads began; a tick with no reads
/// leaves it alone. Time, not tick counts, because an outage can stall one tick for seconds.
struct ReadHealth {
/// Reference point for `failing_since`, because an atomic cannot hold an `Instant`.
base: Instant,
/// Per instance: ms from `base` to the run's first failed read, plus one; zero while healthy.
failing_since: Vec<AtomicU64>,
}
impl ReadHealth {
fn new(instances: usize) -> Self {
Self {
base: Instant::now(),
failing_since: (0..instances).map(|_| AtomicU64::new(0)).collect(),
}
}
fn instance_count(&self) -> usize {
self.failing_since.len()
}
/// Only the background task calls this, one tick at a time per instance, so
/// a plain load and store cannot race.
fn record_tick(&self, idx: usize, any_read_ok: bool, reads_started: Instant) {
let slot = &self.failing_since[idx];
if any_read_ok {
slot.store(0, Ordering::Relaxed);
} else if slot.load(Ordering::Relaxed) == 0 {
let offset_ms = reads_started
.saturating_duration_since(self.base)
.as_millis() as u64;
slot.store(offset_ms + 1, Ordering::Relaxed);
}
}
fn failing_for(&self, idx: usize, now: Instant) -> Option<Duration> {
let since = self.failing_since[idx].load(Ordering::Relaxed);
if since == 0 {
return None;
}
let started = self.base + Duration::from_millis(since - 1);
Some(now.saturating_duration_since(started))
}
}
/// Index of the Redis instance that owns `key`. SipHash-1-3, not `DefaultHasher`, because
/// every pod must agree on the owner across Rust releases or the key's counter splits.
fn instance_index(key: &str, instances: usize) -> usize {
if instances <= 1 {
return 0;
}
let mut hasher = SipHasher13::new();
key.hash(&mut hasher);
(hasher.finish() as usize) % instances
}
/// The client that owns `key`, and its index for metric labels.
fn select_redis_client(
key: &str,
clients: &[Arc<dyn Client + Send + Sync>],
) -> (Arc<dyn Client + Send + Sync>, usize) {
let idx = instance_index(key, clients.len());
(clients[idx].clone(), idx)
}
/// Response returned when a key is rate limited
#[derive(Debug, Clone, PartialEq)]
pub struct GlobalRateLimitResponse {
/// The key that was rate limited
pub key: String,
/// Current effective level (decayed estimate + local pending)
pub current_count: f64,
/// The limit threshold that was exceeded
pub threshold: u64,
/// The sliding window interval
pub window_interval: Duration,
/// Base sync interval; the key's pressure tier scales it
pub sync_interval: Duration,
/// Whether this limit was applied via a custom key override
pub is_custom_limited: bool,
}
/// Reason for failing open (not enforcing rate limit)
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FailOpenReason {
/// Redis read operation timed out
RedisTimeout,
/// Redis returned an error
RedisError,
}
/// Result of evaluating a rate limit check
#[derive(Debug, Clone, PartialEq)]
pub enum EvalResult {
/// Request allowed (under threshold)
Allowed,
/// Request rate limited, includes response metadata
Limited(GlobalRateLimitResponse),
/// Key not subject to rate limiting (custom key mode, unregistered key)
NotApplicable,
/// Failed open due to Redis error or timeout. Not returned today, because
/// Redis failures happen in the background task and never reach a request.
FailOpen { reason: FailOpenReason },
}
/// A distributed rate limiter using a local moka cache with leaky bucket decay,
/// 2-epoch sliding window counters in Redis, and a unified background pipeline
/// for batched reads + writes.
#[derive(Clone)]
pub struct GlobalRateLimiterImpl {
config: GlobalRateLimiterConfig,
cache: Cache<String, CacheEntry>,
update_tx: Option<mpsc::Sender<UpdateRequest>>,
pending_sync: Arc<DashSet<String>>,
scope: &'static str,
/// Drop-to-stop signal for the custom-key refresh task (mirrors `update_tx`).
/// `None` when no `custom_key_source` was configured.
custom_key_refresh_stop: Option<mpsc::Sender<()>>,
/// One entry per Redis instance, indexed like `select_redis_client`.
read_health: Arc<ReadHealth>,
/// How long reads must fail before an instance counts as down; `None` disables the guard.
read_outage_limit: Option<Duration>,
}
#[async_trait]
impl GlobalRateLimiter for GlobalRateLimiterImpl {
async fn check_limit(
&self,
key: &str,
count: u64,
timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
self.check_limit_internal(CheckMode::Global, key, count, timestamp)
.await
}
async fn check_custom_limit(
&self,
key: &str,
count: u64,
timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
self.check_limit_internal(CheckMode::Custom, key, count, timestamp)
.await
}
fn is_custom_key(&self, key: &str) -> bool {
self.config.resolve_custom(key).is_some()
}
fn shutdown(&mut self) {
let _ = self.update_tx.take();
// Dropping the sender closes the refresh task's stop channel.
let _ = self.custom_key_refresh_stop.take();
}
}
impl GlobalRateLimiterImpl {
/// Create a new GlobalRateLimiterImpl
///
/// This spawns a background task for the unified tick loop (reads + writes).
/// Returns an error if `redis_instances` is empty.
pub fn new(
config: GlobalRateLimiterConfig,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
) -> anyhow::Result<Self> {
if redis_instances.is_empty() {
return Err(anyhow::anyhow!(
"GlobalRateLimiterImpl requires at least one Redis instance"
));
}
let scope: &'static str = Box::leak(config.metrics_scope.clone().into_boxed_str());
// An idle timeout shorter than the window would expire entries inside the
// very window they are accumulating counts for, silently under-enforcing.
// Clamp rather than error: this is deploy-time config, and taking capture
// down over a tuning value is worse than running with a corrected one.
let mut config = config;
if config.local_cache_idle_timeout < config.window_interval {
warn!(
scope,
idle_timeout = ?config.local_cache_idle_timeout,
window_interval = ?config.window_interval,
"local_cache_idle_timeout below window_interval would drop counts \
inside the enforcement window; clamping to window_interval"
);
config.local_cache_idle_timeout = config.window_interval;
}
// Same hazard for the TTL: an entry evicted mid-window discards the
// counts it was accumulating, and the next request follows the always-
// allowed miss path.
if config.local_cache_ttl < config.window_interval {
warn!(
scope,
ttl = ?config.local_cache_ttl,
window_interval = ?config.window_interval,
"local_cache_ttl below window_interval would drop counts inside \
the enforcement window; clamping to window_interval"
);
config.local_cache_ttl = config.window_interval;
}
// The previous epoch's Redis key is read for a full window after its
// last write, so a TTL under 2x the window zeroes `prev_count` early
// and under-enforces. The default TTL derives from the default window,
// so a caller that raises only the window lands here. Details in
// GLOBAL_RATE_LIMITER.md.
let min_global_cache_ttl = config.window_interval * 2;
if config.global_cache_ttl < min_global_cache_ttl {
warn!(
scope,
global_cache_ttl = ?config.global_cache_ttl,
window_interval = ?config.window_interval,
"global_cache_ttl below 2x window_interval expires the previous \
epoch key while reads still need it; clamping to 2x window_interval"
);
config.global_cache_ttl = min_global_cache_ttl;
}
let config = config;
let cache = Cache::builder()
.max_capacity(config.local_cache_max_entries)
.time_to_live(config.local_cache_ttl)
.time_to_idle(config.local_cache_idle_timeout)
.eviction_listener(move |_key, _entry: CacheEntry, cause| {
// MUST stay panic-free: moka permanently disables a listener that
// panics. Replaced is an in-place update, not a removal.
if cause != moka::notification::RemovalCause::Replaced {
metrics::counter!(
GLOBAL_RATE_LIMITER_EVICTION_COUNTER,
"scope" => scope,
"cause" => removal_cause_str(cause),
)
.increment(1);
}
})
.build();
let (update_tx, update_rx) = mpsc::channel(config.channel_capacity);
let pending_sync = Arc::new(DashSet::new());
// Spawn the custom-key refresh task when a source is configured. The
// source owns its own reconnect; we only hold a drop-to-stop signal,
// mirroring how `update_tx` closes the tick loop on shutdown.
let custom_key_refresh_stop = config.custom_key_source.clone().map(|source| {
let (stop_tx, stop_rx) = mpsc::channel::<()>(1);
Self::spawn_custom_key_refresh_task(
config.clone(),
source,
config.custom_key_refresh_interval,
stop_rx,
scope,
);
stop_tx
});
let read_health = Arc::new(ReadHealth::new(redis_instances.len()));
let limiter = Self {
read_outage_limit: Self::read_outage_limit(&config),
config: config.clone(),
cache: cache.clone(),
update_tx: Some(update_tx),
pending_sync: pending_sync.clone(),
scope,
custom_key_refresh_stop,
read_health: read_health.clone(),
};
Self::spawn_background_task(
config,
redis_instances,
update_rx,
cache,
pending_sync,
read_health,
scope,
);
Ok(limiter)
}
/// Check if a key is rate limited and enqueue a count update.
///
/// The hot path never touches Redis. Decision is based on local decay estimate.
/// If sync is needed, the entity is pushed to pending_sync for background processing.
async fn check_limit_internal(
&self,
mode: CheckMode,
key: &str,
count: u64,
timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
let threshold = match mode {
CheckMode::Custom => match self.config.resolve_custom(key) {
Some(custom_limit) => custom_limit,
None => return EvalResult::NotApplicable,
},
CheckMode::Global => self.config.global_threshold,
};
let leak_rate = self.config.leak_rate_for(threshold);
let now_instant = Instant::now();
// Enqueue write update to background task
if count > 0 {
let ts = timestamp.unwrap_or_else(Utc::now);
self.enqueue_update(key, count, ts);
}
// Check local cache
let (level, global_level, entry_exists) = if let Some(mut entry) = self.cache.get(key) {
let level = effective_level(&entry, leak_rate, now_instant);
if let Some(synced_at) = entry.synced_at {
let staleness_ms = now_instant.duration_since(synced_at).as_millis() as f64;
metrics::histogram!(GLOBAL_RATE_LIMITER_SYNC_STALENESS_HISTOGRAM, "scope" => self.scope).record(staleness_ms);
}
// Sync decision. The absolute floor is checked first: a key this far
// under its threshold cannot be limited whatever the other nodes
// report, so the round trip buys nothing and the key space is large
// enough that those round trips are the dominant cost. Above the
// floor the pressure tier sets the cadence, and a key that clears the
// floor while still idle-tier syncs on the Low cadence rather than
// never -- otherwise a key that is hot across the fleet but cold on
// any single node would never be discovered.
if self.sync_floor_blocks(level, threshold) {
metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "hit")
.increment(1);
} else {
let effective_pressure = (level / threshold as f64).max(entry.pressure);
let tier_interval =
tier_sync_interval(effective_pressure, self.config.sync_interval)
.unwrap_or_else(|| self.config.sync_interval.mul_f64(4.0));
// A never-synced entry is due immediately: its level is local
// only, and waiting a tier interval to learn the fleet count
// delays every verdict on the key by that long.
let due = match entry.synced_at {
None => true,
Some(synced_at) => now_instant.duration_since(synced_at) > tier_interval,
};
if due {
self.queue_sync(key);
metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "sync_queued")
.increment(1);
} else {
metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "hit")
.increment(1);
}
}
// Increment local_pending and recompute level with this request included
entry.local_pending += count;
let level = effective_level(&entry, leak_rate, now_instant);
let global_level = decayed_global(&entry, leak_rate, now_instant);
self.cache.insert(key.to_string(), entry);
(level, global_level, true)
} else {
// Cache miss: no prior data, allow through and queue a sync at or above the floor
metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "miss").increment(1);
// Insert a fresh entry so subsequent requests have local_pending tracked
let entry = CacheEntry {
estimated_count: 0.0,
synced_at: None,
local_pending: count,
pressure: 0.0,
};
self.cache.insert(key.to_string(), entry);
if !self.sync_floor_blocks(count as f64, threshold) {
self.queue_sync(key);
}
(count as f64, 0.0, false)
};
// While reads fail, `local_pending` grows with nothing to correct it, so judge
// the key by its decayed fleet count alone.
let over_threshold = entry_exists && level >= threshold as f64;
let read_outage = over_threshold && self.read_outage(key, now_instant);
let is_limited = over_threshold && (!read_outage || global_level >= threshold as f64);
if read_outage && !is_limited {
metrics::counter!(GLOBAL_RATE_LIMITER_OUTAGE_LIMIT_SKIPPED_COUNTER, "scope" => self.scope)
.increment(1);
}
if is_limited {
metrics::counter!(GLOBAL_RATE_LIMITER_EVAL_COUNTER, "scope" => self.scope, "result" => "limited").increment(1);
EvalResult::Limited(GlobalRateLimitResponse {
key: key.to_string(),
current_count: level,
threshold,
window_interval: self.config.window_interval,
sync_interval: self.config.sync_interval,
is_custom_limited: mode == CheckMode::Custom,
})
} else {
metrics::counter!(GLOBAL_RATE_LIMITER_EVAL_COUNTER, "scope" => self.scope, "result" => "allowed").increment(1);
EvalResult::Allowed
}
}
/// True, and counted, when `level` is below this key's floor, so a read cannot change the
/// decision. The floor is capped at 1% of the threshold to keep low custom limits enforceable.
fn sync_floor_blocks(&self, level: f64, threshold: u64) -> bool {
if self.config.min_sync_floor == 0 {
return false;
}
let effective_floor = self.config.min_sync_floor.min((threshold / 100).max(1));
if level >= effective_floor as f64 {
return false;
}
metrics::counter!(
GLOBAL_RATE_LIMITER_SYNC_SKIPPED_COUNTER,
"scope" => self.scope,
"reason" => "below_floor",
)
.increment(1);
true
}
fn read_outage_limit(config: &GlobalRateLimiterConfig) -> Option<Duration> {
let limit = config.max_read_outage.unwrap_or(config.window_interval);
(!limit.is_zero()).then_some(limit)
}
fn read_outage(&self, key: &str, now: Instant) -> bool {
let Some(limit) = self.read_outage_limit else {
return false;
};
let idx = instance_index(key, self.read_health.instance_count());
self.read_health
.failing_for(idx, now)
.is_some_and(|failing| failing >= limit)
}
/// Queue a key for background Redis sync, bounded by
/// `max_pending_sync_entries`. A dropped request fails open for one round:
/// the key's next request re-queues it once the backlog drains, and its
/// counts keep flowing to Redis regardless -- only the read is delayed.
fn queue_sync(&self, key: &str) {
if self.pending_sync.len() >= self.config.max_pending_sync_entries
&& !self.pending_sync.contains(key)
{
metrics::counter!(
GLOBAL_RATE_LIMITER_ERROR_COUNTER,
"scope" => self.scope,
"step" => "queue_sync",
"cause" => "pending_sync_full",
)
.increment(1);
return;
}
self.pending_sync.insert(key.to_string());
}
/// Queue an update to be batched and sent to Redis
fn enqueue_update(&self, key: &str, count: u64, timestamp: DateTime<Utc>) {
let update = UpdateRequest {
key: key.to_string(),
count,
timestamp,
};
if let Some(Err(_)) = self.update_tx.as_ref().map(|tx| tx.try_send(update)) {
metrics::counter!(
GLOBAL_RATE_LIMITER_ERROR_COUNTER,
"scope" => self.scope,
"step" => "enqueue_update",
"result" => "error",
"cause" => "channel_full",
)
.increment(1);
}
}
/// Spawn the background task that periodically refreshes custom-key thresholds.
///
/// `tokio::time::interval` fires immediately, so the first fetch happens
/// without delay. The task exits when `stop_rx` closes (all senders dropped
/// via `shutdown`, or when the limiter is dropped).
fn spawn_custom_key_refresh_task(
config: GlobalRateLimiterConfig,
source: Arc<dyn CustomKeyThresholdSource>,
refresh_interval: Duration,
mut stop_rx: mpsc::Receiver<()>,
scope: &'static str,
) {
tokio::spawn(async move {
// `tokio::time::interval` panics on a zero period. A misconfigured
// interval must not kill this (detached) task and silently freeze
// dynamic refreshes, so clamp to a 1s floor and warn instead.
let period = refresh_interval.max(Duration::from_secs(1));
if period != refresh_interval {
warn!(
scope,
?refresh_interval,
"Custom-key refresh interval below 1s floor; clamping to 1s"
);
}
let mut tick = tokio::time::interval(period);
loop {
tokio::select! {
_ = stop_rx.recv() => {
info!(scope, "Custom-key threshold refresh task shutting down");
break;
}
_ = tick.tick() => {
Self::refresh_custom_keys_once(&config, source.as_ref(), scope).await;
}
}
}
});
}
/// Fetch custom-key thresholds once and atomically swap them into the config.
///
/// Only an explicit Redis blob is authoritative: `Ok(Some)` replaces the map
/// (an empty blob — `{}` — is the deliberate clear signal). An absent key
/// (`Ok(None)`) is ambiguous — fresh rollout before the writer runs, an
/// eviction, or an accidental delete — so it is treated as fail-static, the
/// same as `Err`: the current map is kept rather than silently wiping the
/// (fail-open) hot overrides. The source handles its own reconnect, so the
/// next tick retries.
async fn refresh_custom_keys_once(
config: &GlobalRateLimiterConfig,
source: &dyn CustomKeyThresholdSource,
scope: &'static str,
) {
match source.fetch().await {
Ok(Some(map)) => {
let count = map.len();
config.custom_keys.store(Arc::new(map));
metrics::gauge!(CUSTOM_THRESHOLDS_LOADED_GAUGE, "scope" => scope).set(count as f64);
metrics::gauge!(CUSTOM_THRESHOLDS_LAST_REFRESH_GAUGE, "scope" => scope)
.set(Utc::now().timestamp() as f64);
}
Ok(None) => {
// Absent key: keep the current map (fail-static). A deliberate
// clear arrives as an explicit empty blob via the `Ok(Some)` arm;
// the `not_found` counter is already emitted by the source's
// fetch(), so we neither touch the map nor stomp the gauges here.
}
Err(e) => {
error!(
scope,
error = %e,
"Failed to refresh custom-key thresholds, keeping current values"
);
}
}
}
/// Spawn the background loop that handles both reads and writes.
///
/// Between ticks it merges each update into the write batch, and every
/// tick_interval it runs `tick`. The channel is not read while a tick runs.
fn spawn_background_task(
config: GlobalRateLimiterConfig,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
mut update_rx: mpsc::Receiver<UpdateRequest>,
cache: Cache<String, CacheEntry>,
pending_sync: Arc<DashSet<String>>,
read_health: Arc<ReadHealth>,
scope: &'static str,
) {
tokio::spawn(async move {
let mut tick = tokio::time::interval(config.tick_interval);
// Pre-aggregate writes by (key, epoch)
let mut write_batch: HashMap<(String, i64), u64> = HashMap::new();
let mut tick_n: u64 = 0;
let mut update_buf: Vec<UpdateRequest> = Vec::with_capacity(UPDATE_DRAIN_BATCH);
loop {
tokio::select! {
drained = update_rx.recv_many(&mut update_buf, UPDATE_DRAIN_BATCH) => {
if drained == 0 {
// Channel closed: run one last bounded tick and exit
if !write_batch.is_empty() {
Self::tick(
&config, &redis_instances, &cache,
&pending_sync, &mut write_batch, &read_health, scope, tick_n,
).await;
}
break;
}
Self::absorb_requests(&config, &mut write_batch, &mut update_buf, scope);
}
_ = tick.tick() => {
// Drain the queue first, or back-to-back overrun ticks can starve the channel.
let mut queued = update_rx.len();
metrics::gauge!(GLOBAL_RATE_LIMITER_CHANNEL_DEPTH_GAUGE, "scope" => scope)
.set(queued as f64);
while queued > 0 {
let taken = update_rx
.recv_many(&mut update_buf, UPDATE_DRAIN_BATCH.min(queued))
.await;
if taken == 0 {
break;
}
queued -= taken;
Self::absorb_requests(&config, &mut write_batch, &mut update_buf, scope);
}
tick_n = tick_n.wrapping_add(1);
Self::tick(
&config, &redis_instances, &cache,
&pending_sync, &mut write_batch, &read_health, scope, tick_n,
).await;
}
}
}
});
}
fn absorb_requests(
config: &GlobalRateLimiterConfig,
write_batch: &mut HashMap<(String, i64), u64>,
update_buf: &mut Vec<UpdateRequest>,
scope: &'static str,
) {
for req in update_buf.drain(..) {
let epoch = epoch_from_timestamp(req.timestamp, config.window_interval);
Self::absorb_update(
write_batch,
req.key,
epoch,
req.count,
config.max_write_batch_entries,
scope,
);
}
}
/// Merge one update into the deferred write batch, enforcing the entry cap.
///
/// Merges never grow the map, so they are always accepted; only a brand-new
/// (key, epoch) entry can be refused. A refused update undercounts the
/// global tally for that key -- under-enforcement, consistent with every
/// other overload path here failing open -- and is counted so the loss is
/// visible. No log line: at the inflow rates that reach the cap, per-drop
/// logging would itself be a problem.
fn absorb_update(
write_batch: &mut HashMap<(String, i64), u64>,
key: String,
epoch: i64,
count: u64,
max_entries: usize,
scope: &'static str,
) {
let at_cap = write_batch.len() >= max_entries;
match write_batch.entry((key, epoch)) {
std::collections::hash_map::Entry::Occupied(mut entry) => {
*entry.get_mut() += count;
}
std::collections::hash_map::Entry::Vacant(slot) => {
if at_cap {
metrics::counter!(
GLOBAL_RATE_LIMITER_ERROR_COUNTER,
"scope" => scope,
"step" => "enqueue_update",
"cause" => "write_batch_full",
)
.increment(1);
} else {
slot.insert(count);
}
}
}
}
/// Execute one tick of the background pipeline.
///
/// Takes bounded slices of the pending syncs and the write batch, purges
/// stale epochs, then sends chunked writes before chunked reads per instance.
#[allow(clippy::too_many_arguments)]
async fn tick(
config: &GlobalRateLimiterConfig,
redis_instances: &[Arc<dyn Client + Send + Sync>],
cache: &Cache<String, CacheEntry>,
pending_sync: &Arc<DashSet<String>>,
write_batch: &mut HashMap<(String, i64), u64>,
read_health: &ReadHealth,
scope: &'static str,
tick_n: u64,
) {
let tick_start = Instant::now();
// Cache-size gauge every tick (O(1)); per-tier distribution via a
// throttled full scan (slow-moving, see TIER_SCAN_INTERVAL_TICKS).
Self::emit_cache_gauges(cache, scope, tick_n, config.window_interval);
// Take a bounded slice of the pending set rather than all of it. The
// remainder stays queued, so a backlog surfaces as sync staleness instead
// of a tick that overruns its own interval and starves every other key.
// `collect` drops the iterator before the removals, which keeps us off
// dashmap's held-shard-lock path.
let sync_keys: Vec<String> = pending_sync
.iter()
.take(config.max_sync_keys_per_tick)
.map(|r| r.key().clone())
.collect();
for key in &sync_keys {
pending_sync.remove(key);
}
metrics::gauge!(GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE, "scope" => scope)
.set(pending_sync.len() as f64);
// Deferred entries whose epoch has aged out of the readable window can
// no longer affect any decision: reads consult only the current and
// previous epochs. Purge them instead of spending write commands (and
// deferral slots) on counts nothing will ever read.
let min_live_epoch = epoch_from_timestamp(Utc::now(), config.window_interval) - 1;
let before_purge = write_batch.len();
write_batch.retain(|(_, epoch), _| *epoch >= min_live_epoch);
let purged = before_purge - write_batch.len();
if purged > 0 {
metrics::counter!(
GLOBAL_RATE_LIMITER_ERROR_COUNTER,
"scope" => scope,
"step" => "pipeline",
"cause" => "stale_epoch_purged",
)
.increment(purged as u64);
}
// Bound the write drain too, or a burst's write waves eat the whole tick before reads.
// The rest stays batched and lands a few ticks late, unless the purge above drops it.
let writes: HashMap<(String, i64), u64> =
if write_batch.len() <= config.max_sync_keys_per_tick {
std::mem::take(write_batch)
} else {
let drain_keys: Vec<(String, i64)> = write_batch
.keys()
.take(config.max_sync_keys_per_tick)
.cloned()
.collect();
drain_keys
.into_iter()
.filter_map(|k| write_batch.remove_entry(&k))
.collect()
};
metrics::gauge!(GLOBAL_RATE_LIMITER_WRITE_DEFERRED_GAUGE, "scope" => scope)
.set(write_batch.len() as f64);
let read_count = sync_keys.len();
let write_count = writes.len();
if read_count == 0 && write_count == 0 {
return;
}
metrics::histogram!(GLOBAL_RATE_LIMITER_PIPELINE_SIZE_HISTOGRAM, "scope" => scope, "op" => "read")
.record(read_count as f64);
metrics::histogram!(GLOBAL_RATE_LIMITER_PIPELINE_SIZE_HISTOGRAM, "scope" => scope, "op" => "write")
.record(write_count as f64);
metrics::gauge!(GLOBAL_RATE_LIMITER_PENDING_SYNC_SIZE_GAUGE, "scope" => scope)
.set(read_count as f64);
// Partition work by Redis instance
// For simplicity with single-instance (common case), skip partitioning
if redis_instances.len() == 1 {
Self::tick_single_instance(
config,
&redis_instances[0],
0,
cache,
&sync_keys,
&writes,
read_health,
scope,
)
.await;
} else {
Self::tick_multi_instance(
config,
redis_instances,
cache,
&sync_keys,
&writes,
read_health,
scope,
)
.await;
}
metrics::histogram!(GLOBAL_RATE_LIMITER_TICK_HISTOGRAM, "scope" => scope)
.record(tick_start.elapsed().as_micros() as f64 / 1000.0);
}
/// Execute a tick against a single Redis instance (the common case).
#[allow(clippy::too_many_arguments)]
async fn tick_single_instance(
config: &GlobalRateLimiterConfig,
redis: &Arc<dyn Client + Send + Sync>,
redis_idx: usize,
cache: &Cache<String, CacheEntry>,
sync_keys: &[String],
writes: &HashMap<(String, i64), u64>,
read_health: &ReadHealth,
scope: &'static str,
) {
let redis_idx_str: Arc<str> = Arc::from(redis_idx.to_string().as_str());
let now = Utc::now();
// Writes first, then reads. A read result zeroes each synced key's
// `local_pending`, so the read must already include this tick's write
// batch -- reads-first would discard up to a tick of a key's counts
// until its next sync. Writes-first is safe to wait on now that the
// write batch is bounded and chunked: the read delay is capped at a few
// command timeouts, where the old unbounded batch could consume whole
// ticks. Counts deferred past the write cap are still cleared by the
// read before they land; that loss is bounded by the cap and fails
// open, like every other overload path here.
let writes_issued = Self::run_writes(config, redis, &redis_idx_str, writes, scope).await;
let reads_issued = Self::run_reads(
config,
redis,
&redis_idx_str,
cache,
sync_keys,
now,
read_health,
redis_idx,
scope,
)
.await;
metrics::histogram!(GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM, "scope" => scope, "op" => "write")
.record(writes_issued as f64);
metrics::histogram!(GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM, "scope" => scope, "op" => "read")
.record(reads_issued as f64);
}
/// Issue the write half of a tick as size-bounded, concurrently-executed
/// commands. Returns how many commands were issued.
///
/// One oversized command is the failure mode this exists to prevent: the
/// per-command timeout can only be a meaningful budget if the command's size
/// is bounded, otherwise a growing key space silently converts a working
/// timeout into a guaranteed one.
async fn run_writes(
config: &GlobalRateLimiterConfig,
redis: &Arc<dyn Client + Send + Sync>,
redis_idx_str: &Arc<str>,
writes: &HashMap<(String, i64), u64>,
scope: &'static str,
) -> usize {
if writes.is_empty() {
return 0;
}
let write_items: Vec<(String, i64, i64)> = writes
.iter()
.map(|((key, epoch), count)| {
let redis_key = epoch_key(&config.redis_key_prefix, key, *epoch);
let expire_at =
epoch_expire_at(key, *epoch, config.window_interval, config.global_cache_ttl);
(redis_key, *count as i64, expire_at)
})
.collect();
let chunks: Vec<Vec<(String, i64, i64)>> = write_items
.chunks(config.max_keys_per_command.max(1))
.map(|chunk| chunk.to_vec())
.collect();
let issued = chunks.len();
// Waves of `max_concurrent_commands` rather than a `buffer_unordered`
// stream: the stream combinator forces a higher-ranked `Send` bound the
// spawned tick task cannot satisfy, and this keeps the same bound on
// in-flight commands.
for wave in chunks.chunks(config.max_concurrent_commands.max(1)) {
let futures = wave.iter().map(|chunk| {
let redis_idx_str = redis_idx_str.clone();
async move {
let chunk_len = chunk.len();
let started = Instant::now();
match tokio::time::timeout(
config.global_write_timeout,
redis.batch_incr_by_expire_at(chunk.clone()),
)
.await
{
Ok(Ok(_)) => {
metrics::counter!(
GLOBAL_RATE_LIMITER_RECORDS_COUNTER,
"scope" => scope,
"op" => "redis_write",
"redis_idx" => redis_idx_str.clone(),
)
.increment(chunk_len as u64);
metrics::histogram!(
GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM,
"scope" => scope,
"redis_idx" => redis_idx_str.clone(),
)
.record(started.elapsed().as_micros() as f64 / 1000.0);
}
Ok(Err(e)) => {
Self::record_pipeline_error(scope, &redis_idx_str, "redis_write");
warn!(error = %e, records = chunk_len, "Failed to write rate limit batch to Redis");
// A dead MultiplexedConnection never recovers on its
// own; ask the client to rebuild. The client's own
// response timeout also lands here, as a recoverable
// error that does not trigger a rebuild.
if e.is_unrecoverable_error() {
redis.heal().await;
}
}
Err(_) => {
Self::record_pipeline_error(scope, &redis_idx_str, "write_timeout");
warn!(records = chunk_len, "Redis write timeout in pipeline");
}
}
}
});
futures::future::join_all(futures).await;
}
issued
}
/// Issue the read half of a tick as size-bounded, concurrently-executed
/// commands, applying each chunk's results as it lands. Returns how many
/// commands were issued.
#[allow(clippy::too_many_arguments)]
async fn run_reads(
config: &GlobalRateLimiterConfig,
redis: &Arc<dyn Client + Send + Sync>,
redis_idx_str: &Arc<str>,
cache: &Cache<String, CacheEntry>,
sync_keys: &[String],
now: DateTime<Utc>,
read_health: &ReadHealth,
redis_idx: usize,
scope: &'static str,
) -> usize {
if sync_keys.is_empty() {
return 0;
}
// Each entity costs two Redis keys (current + previous epoch), so the
// entity chunk is half the per-command key budget.
let entities_per_chunk = (config.max_keys_per_command / 2).max(1);
let chunks: Vec<&[String]> = sync_keys.chunks(entities_per_chunk).collect();
let issued = chunks.len();
let mut any_read_ok = false;
// Start the outage clock when the reads went out, not when a stalled tick ends.
let reads_started = Instant::now();
// See `run_writes` for why this is waves of `join_all` rather than a
// `buffer_unordered` stream.
for wave in chunks.chunks(config.max_concurrent_commands.max(1)) {
let futures = wave.iter().map(|chunk| {
let redis_idx_str = redis_idx_str.clone();
async move {
let mut mget_keys: Vec<String> = Vec::with_capacity(chunk.len() * 2);
for key in chunk.iter() {
let (curr, prev) =
epoch_keys(&config.redis_key_prefix, key, now, config.window_interval);
mget_keys.push(curr);
mget_keys.push(prev);
}
let started = Instant::now();
match tokio::time::timeout(config.global_read_timeout, redis.mget(mget_keys))
.await
{
Ok(Ok(results)) => {
metrics::counter!(
GLOBAL_RATE_LIMITER_RECORDS_COUNTER,
"scope" => scope,
"op" => "redis_read",
"redis_idx" => redis_idx_str.clone(),
)
.increment(results.len() as u64);
metrics::histogram!(
GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM,
"scope" => scope,
"redis_idx" => redis_idx_str.clone(),
)
.record(started.elapsed().as_micros() as f64 / 1000.0);
Self::process_read_results(config, cache, chunk, &results, now, scope);
true
}
Ok(Err(e)) => {
Self::record_pipeline_error(scope, &redis_idx_str, "redis_error");
warn!(keys = chunk.len(), error = %e, "Failed to read rate limits from Redis");
if e.is_unrecoverable_error() {
redis.heal().await;
}
false
}
Err(_) => {
Self::record_pipeline_error(scope, &redis_idx_str, "read_timeout");
warn!(keys = chunk.len(), "Redis read timeout in pipeline");
false
}
}
}
});
any_read_ok |= futures::future::join_all(futures)
.await
.into_iter()
.any(|ok| ok);
}
// One success means the instance answers, so a tick of mixed results
// counts as healthy.
read_health.record_tick(redis_idx, any_read_ok, reads_started);
let failing_for = read_health
.failing_for(redis_idx, Instant::now())
.unwrap_or_default();
metrics::gauge!(
GLOBAL_RATE_LIMITER_READ_FAILING_SECONDS_GAUGE,
"scope" => scope,
"redis_idx" => redis_idx_str.clone(),
)
.set(failing_for.as_secs_f64());
issued
}
/// Record a pipeline-step failure. `cause` distinguishes read from write so
/// a saturating side is identifiable from the metric alone.
fn record_pipeline_error(scope: &'static str, redis_idx_str: &Arc<str>, cause: &'static str) {
metrics::counter!(
GLOBAL_RATE_LIMITER_ERROR_COUNTER,
"scope" => scope,
"step" => "pipeline",
"cause" => cause,
"redis_idx" => redis_idx_str.clone(),
)
.increment(1);
}
/// Execute a tick partitioned across multiple Redis instances.
async fn tick_multi_instance(
config: &GlobalRateLimiterConfig,
redis_instances: &[Arc<dyn Client + Send + Sync>],
cache: &Cache<String, CacheEntry>,
sync_keys: &[String],
writes: &HashMap<(String, i64), u64>,
read_health: &ReadHealth,
scope: &'static str,
) {
// Partition reads by Redis instance
let mut read_partitions: Vec<Vec<String>> = vec![Vec::new(); redis_instances.len()];
for key in sync_keys {
let (_, idx) = select_redis_client(key, redis_instances);
read_partitions[idx].push(key.clone());
}
// Partition writes by Redis instance
let mut write_partitions: Vec<HashMap<(String, i64), u64>> =
vec![HashMap::new(); redis_instances.len()];
for ((key, epoch), count) in writes {
let (_, idx) = select_redis_client(key, redis_instances);
write_partitions[idx].insert((key.clone(), *epoch), *count);
}
// Execute each partition in parallel
let active_indices: Vec<usize> = (0..redis_instances.len())
.filter(|idx| !read_partitions[*idx].is_empty() || !write_partitions[*idx].is_empty())
.collect();
let futures: Vec<_> = active_indices
.into_iter()
.map(|idx| {
let config = config.clone();
let redis = redis_instances[idx].clone();
let cache = cache.clone();
let reads = std::mem::take(&mut read_partitions[idx]);
let writes_partition = std::mem::take(&mut write_partitions[idx]);
async move {
Self::tick_single_instance(
&config,
&redis,
idx,
&cache,
&reads,
&writes_partition,
read_health,
scope,
)
.await;
}
})
.collect();
futures::future::join_all(futures).await;
}
/// Process MGET results from a read pipeline, updating cache entries.
///
/// Results come in pairs: [current_epoch_value, prev_epoch_value] for each entity.
fn process_read_results(
config: &GlobalRateLimiterConfig,
cache: &Cache<String, CacheEntry>,
sync_keys: &[String],
results: &[Option<Vec<u8>>],
now: DateTime<Utc>,
scope: &'static str,
) {
let now_instant = Instant::now();
for (i, key) in sync_keys.iter().enumerate() {
let base_idx = i * 2;
if base_idx + 1 >= results.len() {
break;
}
let current_count = parse_redis_count(&results[base_idx]);
let prev_count = parse_redis_count(&results[base_idx + 1]);
let estimated = weighted_count(prev_count, current_count, now, config.window_interval);
let threshold = config
.resolve_custom(key)
.unwrap_or(config.global_threshold);
let pressure = estimated / threshold as f64;
let new_tier = PressureTier::from_pressure(pressure);
// Single lookup: emit estimate drift + tier transition for the prior entry.
if let Some(old_entry) = cache.get(key) {
let leak_rate = config.leak_rate_for(threshold);
let local_estimate = effective_level(&old_entry, leak_rate, now_instant);
let drift = (local_estimate - estimated).abs() / threshold as f64;
metrics::histogram!(GLOBAL_RATE_LIMITER_ESTIMATE_DRIFT_HISTOGRAM, "scope" => scope)
.record(drift);
let old_tier = PressureTier::from_pressure(old_entry.pressure);
if old_tier != new_tier {
metrics::counter!(
GLOBAL_RATE_LIMITER_TIER_TRANSITIONS_COUNTER,
"scope" => scope,
"from" => old_tier.as_str(),
"to" => new_tier.as_str(),
)
.increment(1);
}
}
// Redis already holds this node's written events, so reset local_pending to avoid
// double-counting. Events whose write has not landed drop out until a later read.
cache.insert(
key.clone(),
CacheEntry {
estimated_count: estimated,
synced_at: Some(now_instant),
local_pending: 0,
pressure,
},
);
}
}
/// Emit cache observability gauges from the background task (off the hot path).
///
/// `cache_size` is cheap (`entry_count`) so it emits every tick. The per-tier
/// distribution needs a full `cache.iter()` scan, so it runs only every
/// `TIER_SCAN_INTERVAL_TICKS`: the distribution moves slowly and prod metrics
/// dedup to 60s, so scanning every tick would be wasted work under load.
fn emit_cache_gauges(
cache: &Cache<String, CacheEntry>,
scope: &'static str,
tick_n: u64,
window_interval: Duration,
) {
metrics::gauge!(GLOBAL_RATE_LIMITER_CACHE_SIZE_GAUGE, "scope" => scope)
.set(cache.entry_count() as f64);
// Deployments that share a Redis key prefix must agree on the window,
// or their epoch keys diverge and the shared counter silently splits.
// Publish the running value so an alert can compare deployments.
metrics::gauge!(GLOBAL_RATE_LIMITER_WINDOW_GAUGE, "scope" => scope)
.set(window_interval.as_secs_f64());
if tick_n.is_multiple_of(TIER_SCAN_INTERVAL_TICKS) {
let tier_counts = Self::scan_tier_counts(cache);
for i in 0..4 {
metrics::gauge!(GLOBAL_RATE_LIMITER_SYNC_TIER_GAUGE, "scope" => scope, "tier" => TIER_LABELS[i])
.set(tier_counts[i] as f64);
}
}
}
/// Full O(n) scan tallying live entries per pressure tier (indexed by
/// `PressureTier::index()`). Off the hot path; see `TIER_SCAN_INTERVAL_TICKS`.
fn scan_tier_counts(cache: &Cache<String, CacheEntry>) -> [u64; 4] {
let mut tier_counts = [0u64; 4];
for (_, entry) in cache.iter() {
tier_counts[PressureTier::from_pressure(entry.pressure).index()] += 1;
}
tier_counts
}
}
/// Map a Moka removal cause to a stable metric label (Replaced is filtered out
/// before this is called).
fn removal_cause_str(cause: moka::notification::RemovalCause) -> &'static str {
use moka::notification::RemovalCause;
match cause {
RemovalCause::Expired => "expired",
RemovalCause::Explicit => "explicit",
RemovalCause::Size => "size",
RemovalCause::Replaced => "replaced",
}
}
/// Parse a Redis byte response into a u64 count, defaulting to 0
fn parse_redis_count(value: &Option<Vec<u8>>) -> u64 {
value
.as_ref()
.and_then(|bytes| std::str::from_utf8(bytes).ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::custom_key_source::testing::MockCustomKeyThresholdSource;
use common_redis::{CustomRedisError, MockRedisClient};
fn test_config() -> GlobalRateLimiterConfig {
GlobalRateLimiterConfig {
global_threshold: 10,
window_interval: Duration::from_secs(60),
sync_interval: Duration::from_secs(15),
tick_interval: Duration::from_millis(50),
redis_key_prefix: "test:".to_string(),
global_cache_ttl: Duration::from_secs(120),
local_cache_ttl: Duration::from_secs(1),
local_cache_idle_timeout: Duration::from_millis(500),
local_cache_max_entries: 100,
channel_capacity: 100,
custom_keys: Arc::new(ArcSwap::from_pointee(HashMap::new())),
custom_key_resolver: None,
custom_key_source: None,
custom_key_refresh_interval: Duration::from_secs(60),
global_read_timeout: Duration::from_millis(5),
global_write_timeout: Duration::from_millis(10),
metrics_scope: "test".to_string(),
// Tests drive a threshold of 10, so a production-sized floor would
// suppress every sync. 0 keeps the pre-floor behavior; the floor's
// own behavior is covered by the dedicated tests below.
min_sync_floor: 0,
max_read_outage: None,
max_sync_keys_per_tick: 20_000,
max_keys_per_command: 2_000,
max_concurrent_commands: 4,
max_write_batch_entries: 200_000,
max_pending_sync_entries: 200_000,
}
}
/// Seed a config's custom-key map (whole-map swap), mirroring how the
/// dynamic refresh path replaces thresholds at runtime.
fn set_custom_keys(config: &GlobalRateLimiterConfig, pairs: &[(&str, u64)]) {
let map: HashMap<String, u64> = pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect();
config.custom_keys.store(Arc::new(map));
}
// --- Epoch calculation tests (parameterized) ---
#[test]
fn test_epoch_from_timestamp() {
let cases = vec![
// (unix_secs, window_secs, expected_epoch)
(60, 60, 1), // exact boundary
(90, 60, 1), // mid-epoch
(119, 60, 1), // end of epoch
(120, 60, 2), // next epoch
(0, 60, 0), // zero
(1735000080, 60, 28916668), // large timestamp
(30, 30, 1), // different window
];
for (unix, window_secs, expected) in cases {
let ts = DateTime::from_timestamp(unix, 0).unwrap();
let window = Duration::from_secs(window_secs);
assert_eq!(
epoch_from_timestamp(ts, window),
expected,
"epoch_from_timestamp({unix}, {window_secs}s) should be {expected}"
);
}
}
#[test]
fn test_epoch_key_format() {
assert_eq!(epoch_key("prefix", "mykey", 42), "prefix:mykey:42");
}
#[test]
fn test_epoch_keys_returns_current_and_prev() {
let ts = DateTime::from_timestamp(120, 0).unwrap(); // epoch 2 with 60s window
let (curr, prev) = epoch_keys("p", "k", ts, Duration::from_secs(60));
assert_eq!(curr, "p:k:2");
assert_eq!(prev, "p:k:1");
}
#[test]
fn test_epoch_expire_at_never_expires_inside_the_readable_window() {
let window = Duration::from_secs(120);
let ttl = Duration::from_secs(240);
// Epoch 10 is read while the clock sits in epoch 10 or 11, so the key
// must outlive the end of epoch 11 at (10 + 2) * 120.
let last_readable_instant = 12 * 120 - 1;
for key in ["a", "b", "phc_token:distinct", "", "zzzzzzzzzzzz"] {
let at = epoch_expire_at(key, 10, window, ttl);
assert!(
at > last_readable_instant,
"{key} expired at {at}, inside the readable window"
);
}
}
#[test]
fn test_epoch_expire_at_spreads_keys_across_a_window() {
let window = Duration::from_secs(120);
let ttl = Duration::from_secs(240);
let base = 12 * 120;
// Without this spread a whole generation expires in one instant, and
// that burst blocks Redis long enough to time out concurrent writes.
let deadlines: std::collections::HashSet<i64> = (0..500)
.map(|i| epoch_expire_at(&format!("key{i}"), 10, window, ttl))
.collect();
assert!(
deadlines.len() > 100,
"expected keys to land on many distinct deadlines, got {}",
deadlines.len()
);
assert!(deadlines.iter().all(|d| *d >= base && *d < base + 120));
// The offset is stable per key, so a repeat write cannot move a
// deadline and leave the key alive longer each time.
assert_eq!(
epoch_expire_at("k", 10, window, ttl),
epoch_expire_at("k", 10, window, ttl)
);
}
fn mock_pool(n: usize) -> Vec<Arc<dyn Client + Send + Sync>> {
(0..n)
.map(|_| Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>)
.collect()
}
#[test]
fn test_select_redis_client_assignment_is_pinned() {
// Golden assignments. Every pod must route a key to the same instance,
// so a change of hash algorithm splits one entity's counter across two
// instances and under-enforces its limit with nothing in the metrics to
// show it. Pin the mapping so such a change fails here instead.
let pool = mock_pool(4);
let got: Vec<usize> = ["team_1", "team_2", "team_3", "phc_tok:distinct"]
.iter()
.map(|k| select_redis_client(k, &pool).1)
.collect();
assert_eq!(got, vec![2, 2, 3, 1]);
}
#[test]
fn test_select_redis_client_spreads_across_the_pool() {
for n in [2usize, 3, 4, 8] {
let pool = mock_pool(n);
let mut counts = vec![0usize; n];
for i in 0..4000 {
counts[select_redis_client(&format!("key{i}"), &pool).1] += 1;
}
let ideal = 4000 / n;
for (shard, c) in counts.iter().enumerate() {
assert!(
*c > ideal / 2 && *c < ideal * 2,
"pool of {n}: shard {shard} got {c}, ideal {ideal}"
);
}
}
}
#[test]
fn test_select_redis_client_single_instance_skips_hashing() {
let pool = mock_pool(1);
assert_eq!(select_redis_client("anything", &pool).1, 0);
}
#[test]
fn test_epoch_expire_at_spread_survives_sharding() {
// `select_redis_client` shards on a hash of the same key. If the expiry
// offset used that hash too, a shard count sharing a factor with the
// window would leave each shard only a fraction of the offsets, and the
// expiry burst would come back per shard.
let window = Duration::from_secs(120);
let ttl = Duration::from_secs(240);
for shards in [2usize, 3, 4, 8] {
let mut per_shard: std::collections::HashMap<usize, std::collections::HashSet<i64>> =
std::collections::HashMap::new();
for i in 0..4000 {
let key = format!("key{i}");
let shard = instance_index(&key, shards);
per_shard
.entry(shard)
.or_default()
.insert(epoch_expire_at(&key, 10, window, ttl));
}
for (shard, offsets) in &per_shard {
assert!(
offsets.len() > 100,
"shard {shard} of {shards} saw only {} distinct deadlines",
offsets.len()
);
}
}
}
#[test]
fn test_epoch_expire_at_carries_configured_slack_as_grace() {
let window = Duration::from_secs(120);
let base = 12 * 120;
// TTL above the two-window minimum becomes clock-skew grace, on top of
// the per-key spread.
assert!(epoch_expire_at("k", 10, window, Duration::from_secs(250)) >= base + 10);
// A TTL below the minimum cannot pull the deadline inside the window.
assert!(epoch_expire_at("k", 10, window, Duration::from_secs(60)) >= base);
}
// --- Weighted count estimation tests (parameterized) ---
#[test]
fn test_weighted_count_estimation() {
let cases = vec![
// (prev, current, unix_secs, window_secs, expected_approx)
(100, 0, 60, 60, 100.0), // progress=0.0: full prev weight
(100, 0, 90, 60, 50.0), // progress=0.5: half prev weight
(100, 50, 90, 60, 100.0), // progress=0.5: 50 + 50
(0, 100, 90, 60, 100.0), // prev=0, all current
(0, 0, 90, 60, 0.0), // both zero
(100, 100, 60, 60, 200.0), // progress=0.0: full prev + current
];
for (prev, current, unix, window_secs, expected) in cases {
let ts = DateTime::from_timestamp(unix, 0).unwrap();
let window = Duration::from_secs(window_secs);
let result = weighted_count(prev, current, ts, window);
assert!(
(result - expected).abs() < 0.01,
"weighted_count({prev}, {current}, t={unix}, w={window_secs}) = {result}, expected {expected}"
);
}
}
// --- Leaky bucket decay tests (parameterized) ---
#[test]
fn test_effective_level_decay() {
let base = Instant::now();
let cases = vec![
// (estimated_count, elapsed_secs, local_pending, leak_rate, synced, expected)
(100.0, 0.0, 0, 10.0, true, 100.0), // no elapsed: full count
(100.0, 10.0, 0, 10.0, true, 0.0), // full drain
(100.0, 5.0, 0, 10.0, true, 50.0), // partial drain
(100.0, 0.0, 50, 10.0, true, 150.0), // local_pending adds
(100.0, 5.0, 30, 10.0, true, 80.0), // drain + pending: (100-50)+30
(10.0, 20.0, 0, 10.0, true, 0.0), // over-drain floors at 0
(10.0, 20.0, 5, 10.0, true, 5.0), // over-drain + pending
// Never synced: no fleet estimate exists yet, so nothing decays and
// the level is the local count alone, whatever the other fields say.
(100.0, 10.0, 7, 10.0, false, 7.0),
(0.0, 0.0, 0, 10.0, false, 0.0),
];
for (est, elapsed, pending, rate, synced, expected) in cases {
let entry = CacheEntry {
estimated_count: est,
synced_at: synced.then_some(base),
local_pending: pending,
pressure: 0.0,
};
let now = base + Duration::from_secs_f64(elapsed);
let result = effective_level(&entry, rate, now);
assert!(
(result - expected).abs() < 0.01,
"effective_level(est={est}, elapsed={elapsed}s, pending={pending}, rate={rate}, synced={synced}) = {result}, expected {expected}"
);
}
}
// --- Pressure tier tests (parameterized) ---
#[test]
fn test_pressure_tier_from_pressure() {
let cases = vec![
(0.0, PressureTier::Idle),
(0.05, PressureTier::Idle),
(0.09, PressureTier::Idle),
(0.1, PressureTier::Low),
(0.25, PressureTier::Low),
(0.49, PressureTier::Low),
(0.5, PressureTier::Normal),
(0.75, PressureTier::Normal),
(0.79, PressureTier::Normal),
(0.8, PressureTier::Hot),
(0.95, PressureTier::Hot),
(1.0, PressureTier::Hot),
(1.5, PressureTier::Hot),
];
for (pressure, expected) in cases {
assert_eq!(
PressureTier::from_pressure(pressure),
expected,
"PressureTier::from_pressure({pressure}) should be {expected:?}"
);
}
}
#[test]
fn test_tier_sync_interval() {
let base = Duration::from_secs(15);
let cases = vec![
// (pressure, expected_multiplier_of_base)
(0.05, None), // Idle: skip
(0.25, Some(Duration::from_secs(60))), // Low: 4x
(0.65, Some(Duration::from_secs(15))), // Normal: 1x
(0.9, Some(Duration::from_millis(7500))), // Hot: 0.5x
];
for (pressure, expected) in cases {
let result = tier_sync_interval(pressure, base);
assert_eq!(
result, expected,
"tier_sync_interval({pressure}, 15s) should be {expected:?}, got {result:?}"
);
}
}
// --- Config tests ---
#[test]
fn test_config_defaults() {
let config = GlobalRateLimiterConfig::default();
assert_eq!(config.global_threshold, 1_000_000);
assert_eq!(config.window_interval, Duration::from_secs(60));
assert_eq!(config.sync_interval, Duration::from_secs(15));
assert_eq!(config.tick_interval, Duration::from_secs(1));
assert_eq!(config.redis_key_prefix, "@posthog/global_rate_limiter");
assert_eq!(config.global_cache_ttl, Duration::from_secs(120));
assert_eq!(config.local_cache_ttl, Duration::from_secs(600));
assert_eq!(config.local_cache_idle_timeout, Duration::from_secs(300));
assert_eq!(config.global_read_timeout, Duration::from_millis(250));
assert_eq!(config.global_write_timeout, Duration::from_millis(250));
assert_eq!(config.local_cache_max_entries, 300_000);
assert_eq!(config.channel_capacity, 1_000_000);
assert_eq!(config.min_sync_floor, 10);
assert_eq!(config.max_sync_keys_per_tick, 20_000);
assert_eq!(config.max_keys_per_command, 2_000);
assert_eq!(config.max_concurrent_commands, 4);
assert_eq!(config.max_write_batch_entries, 200_000);
assert_eq!(config.max_pending_sync_entries, 200_000);
assert!(config.custom_keys.load().is_empty());
assert!(config.custom_key_resolver.is_none());
assert_eq!(config.metrics_scope, "default");
}
#[test]
fn test_leak_rate() {
let config = test_config(); // threshold=10, window=60s
assert!((config.leak_rate() - 10.0 / 60.0).abs() < 0.0001);
assert!((config.leak_rate_for(100) - 100.0 / 60.0).abs() < 0.0001);
}
// --- Limiter behavior tests ---
#[tokio::test]
async fn test_not_limited_when_under_threshold() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"test_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 0.5,
},
);
let result = limiter.check_limit("test_key", 1, None).await;
assert!(
matches!(result, EvalResult::Allowed),
"Should return Allowed when under threshold, got {result:?}"
);
}
#[tokio::test]
async fn test_limited_when_at_threshold() {
let client = Arc::new(MockRedisClient::new());
let config = test_config(); // threshold = 10
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"test_key".to_string(),
CacheEntry {
estimated_count: 10.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 1.0,
},
);
let result = limiter.check_limit("test_key", 1, None).await;
assert!(
matches!(result, EvalResult::Limited(_)),
"Should be Limited when at/over threshold, got {result:?}"
);
}
#[tokio::test]
async fn test_rate_limit_response_fields() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"test_key".to_string(),
CacheEntry {
estimated_count: 15.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 1.5,
},
);
let result = limiter.check_limit("test_key", 1, None).await;
let response = match result {
EvalResult::Limited(r) => r,
other => panic!("Expected Limited, got {other:?}"),
};
assert_eq!(response.key, "test_key");
assert!(response.current_count >= 15.0);
assert_eq!(response.threshold, 10);
assert_eq!(response.window_interval, Duration::from_secs(60));
assert_eq!(response.sync_interval, Duration::from_secs(15));
assert!(!response.is_custom_limited);
}
#[tokio::test]
async fn test_cache_miss_returns_allowed() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// No cache entry: first request should be allowed and entity queued for sync
let result = limiter.check_limit("unknown_key", 1, None).await;
assert!(
matches!(result, EvalResult::Allowed),
"Cache miss should return Allowed, got {result:?}"
);
// Verify entity was queued for sync
assert!(
limiter.pending_sync.contains("unknown_key"),
"Should have queued entity for sync"
);
}
#[tokio::test]
async fn test_cache_hit_no_redis_calls() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config.clone(), vec![client.clone()]).unwrap();
// Fresh cache entry (synced_at = now, well within sync_interval)
limiter.cache.insert(
"cached_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 0.5,
},
);
let result = limiter.check_limit("cached_key", 1, None).await;
assert!(
matches!(result, EvalResult::Allowed),
"Should return Allowed with cached count of 5, got {result:?}"
);
// No mget calls should have been made (decision was local)
let calls = client.get_calls();
let mget_calls: Vec<_> = calls.iter().filter(|c| c.op == "mget").collect();
assert!(
mget_calls.is_empty(),
"Should not have called mget when cache hit"
);
}
#[tokio::test]
async fn test_local_pending_incremented_on_check() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"key_a".to_string(),
CacheEntry {
estimated_count: 1.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 0.1,
},
);
let _ = limiter.check_limit("key_a", 5, None).await;
let entry = limiter.cache.get("key_a").unwrap();
assert_eq!(
entry.local_pending, 5,
"local_pending should be incremented by count"
);
}
#[tokio::test]
async fn test_update_queued_even_when_limited() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client.clone()]).unwrap();
limiter.cache.insert(
"limited_key".to_string(),
CacheEntry {
estimated_count: 10.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 1.0,
},
);
let result = limiter.check_limit("limited_key", 1, None).await;
assert!(
matches!(result, EvalResult::Limited(_)),
"Should be Limited, got {result:?}"
);
// Give background task time to process the tick
tokio::time::sleep(Duration::from_millis(100)).await;
let calls = client.get_calls();
let batch_calls: Vec<_> = calls
.iter()
.filter(|c| c.op == "batch_incr_by_expire_at")
.collect();
assert!(
!batch_calls.is_empty(),
"Should have queued update to Redis"
);
}
// --- Custom key tests ---
#[tokio::test]
async fn test_custom_mode_unknown_key_returns_not_applicable() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
set_custom_keys(&config, &[("known_key", 5)]);
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
let result = limiter.check_custom_limit("unknown_key", 100, None).await;
assert!(
matches!(result, EvalResult::NotApplicable),
"Custom mode should return NotApplicable for unknown keys, got {result:?}"
);
}
#[tokio::test]
async fn test_custom_mode_uses_custom_limit() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
set_custom_keys(&config, &[("custom_key", 5)]);
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"custom_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 1.0,
},
);
let result = limiter.check_custom_limit("custom_key", 1, None).await;
let response = match result {
EvalResult::Limited(r) => r,
other => panic!("Should be Limited when reaching custom limit, got {other:?}"),
};
assert!(response.is_custom_limited);
}
#[tokio::test]
async fn test_custom_mode_under_custom_limit() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
set_custom_keys(&config, &[("custom_key", 10)]);
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"custom_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 0.5,
},
);
let result = limiter.check_custom_limit("custom_key", 1, None).await;
assert!(
matches!(result, EvalResult::Allowed),
"Should return Allowed when under custom limit, got {result:?}"
);
}
#[tokio::test]
async fn test_is_custom_key() {
let client = Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>;
let config = test_config();
set_custom_keys(&config, &[("registered", 42)]);
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert!(limiter.is_custom_key("registered"));
assert!(!limiter.is_custom_key("unknown"));
assert!(!limiter.is_custom_key(""));
}
#[tokio::test]
async fn test_is_custom_key_empty_map() {
let client = Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>;
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert!(!limiter.is_custom_key("anything"));
}
#[tokio::test]
async fn test_custom_key_behavior() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
set_custom_keys(&config, &[("custom_a", 5), ("custom_b", 10)]);
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
limiter.cache.insert(
"custom_a".to_string(),
CacheEntry {
estimated_count: 10.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 2.0,
},
);
let result = limiter.check_custom_limit("custom_a", 1, None).await;
assert!(
matches!(result, EvalResult::Limited(_)),
"custom_a should be Limited, got {result:?}"
);
let result = limiter.check_custom_limit("unknown_key", 1, None).await;
assert!(
matches!(result, EvalResult::NotApplicable),
"unknown_key should return NotApplicable, got {result:?}"
);
let result = limiter.check_custom_limit("", 1, None).await;
assert!(
matches!(result, EvalResult::NotApplicable),
"empty key should return NotApplicable, got {result:?}"
);
}
// --- Sync scheduling tests ---
#[tokio::test]
async fn test_sync_queued_when_interval_exceeded() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// Entry synced long ago: should trigger sync
limiter.cache.insert(
"stale_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now() - Duration::from_secs(60)),
local_pending: 0,
pressure: 0.5,
},
);
let _ = limiter.check_limit("stale_key", 1, None).await;
assert!(
limiter.pending_sync.contains("stale_key"),
"Should have queued stale entity for sync"
);
}
#[tokio::test]
async fn test_sync_not_queued_when_fresh() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// Freshly synced, normal pressure
limiter.cache.insert(
"fresh_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now()),
local_pending: 0,
pressure: 0.5,
},
);
let _ = limiter.check_limit("fresh_key", 1, None).await;
assert!(
!limiter.pending_sync.contains("fresh_key"),
"Should NOT have queued fresh entity for sync"
);
}
/// `test_config` with the sync floor set and the background drain parked, so
/// `pending_sync` assertions observe only what `check_limit` queued.
fn config_with_floor(floor: u64) -> GlobalRateLimiterConfig {
GlobalRateLimiterConfig {
min_sync_floor: floor,
tick_interval: Duration::from_secs(3600),
..test_config()
}
}
#[tokio::test]
async fn test_min_sync_floor_gates_cold_miss_sync() {
// (floor, count, expect_queued)
let cases = vec![
(0, 1, true), // floor disabled: every miss syncs (pre-floor behavior)
(5, 1, false), // below floor: no round trip for a key that cannot be limited
(5, 5, true), // exactly at the floor
(5, 9, true), // above the floor
];
for (floor, count, expect_queued) in cases {
let client = Arc::new(MockRedisClient::new());
// Threshold large enough (floor * 100 or more) that the 1% cap does
// not reduce the configured floor; the cap has its own test below.
let config = GlobalRateLimiterConfig {
global_threshold: 1000,
..config_with_floor(floor)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
let key = format!("cold_{floor}_{count}");
limiter.check_limit(&key, count, None).await;
assert_eq!(
limiter.pending_sync.contains(&key),
expect_queued,
"floor={floor} count={count} should queue sync = {expect_queued}"
);
}
}
#[tokio::test]
async fn test_sync_due_immediately_only_when_never_synced() {
// (label, synced_at, expect_queued)
let cases = vec![
("never synced", None, true),
("synced just now", Some(Instant::now()), false),
];
for (label, synced_at, expect_queued) in cases {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
// 1% of the threshold is 10, so the configured floor of 5 applies.
global_threshold: 1000,
min_sync_floor: 5,
// Park the background drain so pending_sync shows only what
// check_limit queued.
tick_interval: Duration::from_secs(3600),
..test_config()
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// local_pending is above the floor, so the sync decision comes down
// to synced_at alone. Pressure stays Idle, whose tier interval is
// 4 x sync_interval = 60s, far longer than this test takes.
limiter.cache.insert(
"above_floor".to_string(),
CacheEntry {
estimated_count: 0.0,
synced_at,
local_pending: 11,
pressure: 0.0,
},
);
let _ = limiter.check_limit("above_floor", 1, None).await;
assert_eq!(
limiter.pending_sync.contains("above_floor"),
expect_queued,
"an entry {label} and above the floor should queue a sync={expect_queued} -- \
a never-synced entry has no fleet count behind it, so making it wait out a \
tier interval hides the key from the limiter for that long"
);
}
}
#[tokio::test]
async fn test_idle_timeout_clamped_up_to_window_interval() {
// (idle_timeout_secs, expected_secs)
let cases = vec![
(10, 60), // below the 60s window: clamped up
(59, 60), // just below: clamped up
(60, 60), // exactly at the window: untouched
(300, 300), // above: untouched
];
for (idle_secs, expected_secs) in cases {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
local_cache_idle_timeout: Duration::from_secs(idle_secs),
..test_config()
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert_eq!(
limiter.config.local_cache_idle_timeout,
Duration::from_secs(expected_secs),
"idle_timeout={idle_secs}s against a 60s window should resolve to {expected_secs}s -- \
an idle timeout inside the window expires entries mid-window and silently under-enforces"
);
}
}
#[tokio::test]
async fn test_global_cache_ttl_clamped_up_to_two_windows() {
// (configured_ttl_secs, expected_secs) against test_config()'s 60s window
let cases = vec![
(30, 120), // below one window
(60, 120), // one window: still expires the previous epoch early
(119, 120), // just below the bound
(120, 120), // exactly at the bound: untouched
(600, 600), // above: untouched
];
for (ttl_secs, expected_secs) in cases {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
global_cache_ttl: Duration::from_secs(ttl_secs),
..test_config()
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert_eq!(
limiter.config.global_cache_ttl,
Duration::from_secs(expected_secs),
"global_cache_ttl={ttl_secs}s against a 60s window should resolve to {expected_secs}s -- \
a TTL under 2x the window expires the previous epoch key while reads still \
need it, so weighted_count takes prev_count as 0 and under-enforces"
);
}
}
#[tokio::test]
async fn test_sync_floor_capped_at_one_percent_of_threshold() {
// (configured_floor, threshold, count, expect_queued)
let cases = vec![
// Custom-style low threshold: cap = max(1, 100/100) = 1, so any
// counted event syncs. A floor tuned for the global threshold must
// not make a low custom override unenforceable.
(10, 100, 1, true),
// Threshold 500: cap = 5. The configured 10 is reduced to 5.
(10, 500, 4, false),
(10, 500, 5, true),
// Large threshold: cap = 150 leaves the configured 10 in charge.
(10, 15_000, 9, false),
(10, 15_000, 10, true),
];
for (floor, threshold, count, expect_queued) in cases {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
global_threshold: threshold,
..config_with_floor(floor)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
let key = format!("cap_{floor}_{threshold}_{count}");
limiter.check_limit(&key, count, None).await;
assert_eq!(
limiter.pending_sync.contains(&key),
expect_queued,
"floor={floor} threshold={threshold} count={count} should queue sync = {expect_queued}"
);
}
}
#[tokio::test]
async fn test_ttl_clamped_up_to_window_interval() {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
local_cache_ttl: Duration::from_secs(1),
..test_config() // 60s window
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert_eq!(
limiter.config.local_cache_ttl,
Duration::from_secs(60),
"a TTL below the window evicts entries mid-window; the next request \
takes the always-allowed miss path and the limiter under-enforces"
);
}
#[tokio::test]
async fn test_tick_runs_writes_before_reads() {
let mock = Arc::new(MockRedisClient::new());
let client: Arc<dyn Client + Send + Sync> = mock.clone();
let config = config_with_floor(0);
let cache: Cache<String, CacheEntry> = Cache::builder().max_capacity(100).build();
let pending: Arc<DashSet<String>> = Arc::new(DashSet::new());
pending.insert("read_key".to_string());
let mut writes: HashMap<(String, i64), u64> = HashMap::new();
// Current epoch: a stale epoch would be purged before the write runs.
let epoch = epoch_from_timestamp(Utc::now(), config.window_interval);
writes.insert(("write_key".to_string(), epoch), 5);
GlobalRateLimiterImpl::tick(
&config,
std::slice::from_ref(&client),
&cache,
&pending,
&mut writes,
&ReadHealth::new(1),
"test",
1,
)
.await;
let calls = mock.get_calls();
let first_read = calls.iter().position(|c| c.op == "mget");
let first_write = calls.iter().position(|c| c.op.starts_with("batch_incr"));
assert!(
first_read.is_some() && first_write.is_some(),
"tick should issue both a read and a write"
);
assert!(
first_write < first_read,
"writes must land before reads: the read result zeroes each synced \
key's local_pending, so a read that predates this tick's writes \
silently discards those counts until the next sync. calls={calls:?}"
);
}
#[tokio::test]
async fn test_pending_sync_cap_drops_new_keys() {
// Keys clearing the sync floor faster than the per-tick drain must not
// grow pending_sync without bound; at the cap, new sync requests drop
// (the key re-queues on its next request) while known keys stay queued.
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
max_pending_sync_entries: 2,
..config_with_floor(0)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
for key in ["a", "b", "c", "d"] {
limiter.check_limit(key, 1, None).await;
}
assert_eq!(
limiter.pending_sync.len(),
2,
"pending_sync must stop growing at max_pending_sync_entries"
);
}
#[tokio::test]
async fn test_write_batch_cap_drops_new_keys_but_merges_existing() {
// At the cap, an update for a brand-new key is dropped (bounded memory
// beats an unbounded map under unique-key floods), while an update for
// a key already in the batch still merges -- merging costs no memory
// and dropping it would silently undercount a key we are tracking.
let mut batch: HashMap<(String, i64), u64> = HashMap::new();
batch.insert(("k1".to_string(), 1), 5);
batch.insert(("k2".to_string(), 1), 5);
GlobalRateLimiterImpl::absorb_update(&mut batch, "k3".to_string(), 1, 7, 2, "test");
assert_eq!(batch.len(), 2, "new key at cap must be dropped");
assert!(!batch.contains_key(&("k3".to_string(), 1)));
GlobalRateLimiterImpl::absorb_update(&mut batch, "k1".to_string(), 1, 7, 2, "test");
assert_eq!(
batch.get(&("k1".to_string(), 1)),
Some(&12),
"existing key at cap must still merge"
);
}
#[tokio::test]
async fn test_tick_purges_stale_epochs_instead_of_writing_them() {
let mock = Arc::new(MockRedisClient::new());
let client: Arc<dyn Client + Send + Sync> = mock.clone();
let config = config_with_floor(0); // 60s window
let cache: Cache<String, CacheEntry> = Cache::builder().max_capacity(100).build();
let pending: Arc<DashSet<String>> = Arc::new(DashSet::new());
let current_epoch = epoch_from_timestamp(Utc::now(), config.window_interval);
let mut writes: HashMap<(String, i64), u64> = HashMap::new();
writes.insert(("live".to_string(), current_epoch), 1);
writes.insert(("stale".to_string(), current_epoch - 5), 1);
GlobalRateLimiterImpl::tick(
&config,
std::slice::from_ref(&client),
&cache,
&pending,
&mut writes,
&ReadHealth::new(1),
"test",
1,
)
.await;
let write_calls: Vec<String> = mock
.get_calls()
.into_iter()
.filter(|c| c.op == "batch_incr_by_expire_at")
.map(|c| c.key)
.collect();
let live_key = epoch_key(&config.redis_key_prefix, "live", current_epoch);
let live_expire_at = epoch_expire_at(
"live",
current_epoch,
config.window_interval,
config.global_cache_ttl,
);
assert_eq!(
write_calls,
vec![format!("{live_key}=1@{live_expire_at}")],
"only the readable-epoch entry may be written; a stale epoch can never be read (reads consult current + previous only) and must not spend write commands"
);
assert!(
writes.is_empty(),
"stale entry must be purged, not deferred"
);
}
#[tokio::test]
async fn test_tick_bounds_write_drain_and_carries_remainder() {
let client: Arc<dyn Client + Send + Sync> = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
max_sync_keys_per_tick: 10,
..config_with_floor(0)
};
let cache: Cache<String, CacheEntry> = Cache::builder().max_capacity(100).build();
let pending: Arc<DashSet<String>> = Arc::new(DashSet::new());
let mut writes: HashMap<(String, i64), u64> = HashMap::new();
let epoch = epoch_from_timestamp(Utc::now(), config.window_interval);
for i in 0..25 {
writes.insert((format!("w{i}"), epoch), 1);
}
GlobalRateLimiterImpl::tick(
&config,
std::slice::from_ref(&client),
&cache,
&pending,
&mut writes,
&ReadHealth::new(1),
"test",
1,
)
.await;
assert_eq!(
writes.len(),
15,
"tick must drain at most max_sync_keys_per_tick write entries and \
leave the remainder batched -- deferring keeps counts (they merge by \
key+epoch and land a tick late), dropping them would lose counts"
);
}
/// Poll until the outcome shows. The tick loop runs on real time, so a test waits for
/// what it can observe instead of a guessed delay.
macro_rules! wait_until {
($why:expr, $cond:expr) => {{
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
while !$cond {
assert!(tokio::time::Instant::now() < deadline, "{}", $why);
tokio::time::sleep(Duration::from_millis(5)).await;
}
}};
}
/// The read-outage guard engages on the first failed read, so a test waits for ticks,
/// not for a window of wall-clock time.
fn outage_config(threshold: u64) -> GlobalRateLimiterConfig {
GlobalRateLimiterConfig {
global_threshold: threshold,
max_read_outage: Some(Duration::from_nanos(1)),
tick_interval: Duration::from_millis(10),
..config_with_floor(0)
}
}
fn limited(result: EvalResult) -> bool {
matches!(result, EvalResult::Limited(_))
}
fn written_total(mock: &MockRedisClient) -> u64 {
mock.get_calls()
.iter()
.filter(|c| c.op == "batch_incr_by_expire_at")
.flat_map(|c| c.key.split(','))
.filter_map(|item| {
item.split_once('=')?
.1
.split('@')
.next()?
.parse::<u64>()
.ok()
})
.sum()
}
#[tokio::test(flavor = "current_thread")]
async fn test_ticks_longer_than_the_interval_do_not_starve_the_channel() {
let mock = Arc::new(
MockRedisClient::new().batch_incr_by_expire_at_block(Duration::from_millis(50)),
);
let config = GlobalRateLimiterConfig {
channel_capacity: 2_000,
tick_interval: Duration::from_millis(10),
global_write_timeout: Duration::from_secs(1),
..config_with_floor(0)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![mock.clone()]).unwrap();
// Writes outlast the tick interval, so every tick overruns.
for round in 0..60 {
for i in 0..200 {
limiter.check_limit(&format!("r{round}k{i}"), 1, None).await;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
wait_until!(
"ticks that overran their interval starved the channel and lost updates",
written_total(&mock) == 12_000
);
}
#[tokio::test(flavor = "current_thread")]
async fn test_burst_larger_than_one_poll_budget_is_not_dropped() {
let mock = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
channel_capacity: 400,
tick_interval: Duration::from_millis(10),
..config_with_floor(0)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![mock.clone()]).unwrap();
for i in 0..400 {
limiter.check_limit(&format!("first{i}"), 1, None).await;
}
// One poll of the background task; a single `recv` would take only 128 updates.
tokio::task::yield_now().await;
for i in 0..300 {
limiter.check_limit(&format!("second{i}"), 1, None).await;
}
wait_until!(
"a burst larger than one poll overflowed the channel and lost updates",
written_total(&mock) == 700
);
}
fn failing_reads() -> Arc<dyn Client + Send + Sync> {
Arc::new(MockRedisClient::new().mget_error(CustomRedisError::Timeout))
}
async fn wait_for_read_outage(limiter: &GlobalRateLimiterImpl, threshold: u64) {
let _ = limiter.check_limit("outage_probe", threshold, None).await;
wait_until!(
"reads keep failing, so the guard must engage",
!limited(limiter.check_limit("outage_probe", 1, None).await)
);
}
struct FleetCountCase {
name: &'static str,
reads_failing: bool,
last_read: Option<Instant>,
fleet_count_at_last_read: f64,
events_counted_here_since: u64,
expect_limited: bool,
}
#[tokio::test]
async fn test_read_outage_judges_a_key_by_its_fleet_count_alone() {
let now = Instant::now();
let cases = [
FleetCountCase {
name: "reads failing, never read, this pod's count alone reaches the limit",
reads_failing: true,
last_read: None,
fleet_count_at_last_read: 0.0,
events_counted_here_since: 10_000,
expect_limited: false,
},
FleetCountCase {
name: "reads failing, fleet count under, this pod's count pushes over",
reads_failing: true,
last_read: Some(now),
fleet_count_at_last_read: 5_000.0,
events_counted_here_since: 10_000,
expect_limited: false,
},
FleetCountCase {
name: "reads failing, fleet count itself over",
reads_failing: true,
last_read: Some(now),
fleet_count_at_last_read: 12_000.0,
events_counted_here_since: 0,
expect_limited: true,
},
FleetCountCase {
name: "reads healthy, never read, this pod's count alone reaches the limit",
reads_failing: false,
last_read: None,
fleet_count_at_last_read: 0.0,
events_counted_here_since: 10_000,
expect_limited: true,
},
FleetCountCase {
name: "reads healthy, fleet count under, this pod's count pushes over",
reads_failing: false,
last_read: Some(now),
fleet_count_at_last_read: 5_000.0,
events_counted_here_since: 10_000,
expect_limited: true,
},
];
for case in cases {
let redis: Arc<dyn Client + Send + Sync> = if case.reads_failing {
failing_reads()
} else {
Arc::new(MockRedisClient::new())
};
let limiter = GlobalRateLimiterImpl::new(outage_config(10_000), vec![redis]).unwrap();
if case.reads_failing {
wait_for_read_outage(&limiter, 10_000).await;
}
limiter.cache.insert(
"key".to_string(),
CacheEntry {
estimated_count: case.fleet_count_at_last_read,
synced_at: case.last_read,
local_pending: case.events_counted_here_since,
pressure: 0.0,
},
);
let result = limiter.check_limit("key", 1, None).await;
assert_eq!(
limited(result),
case.expect_limited,
"{}: while reads fail, only the fleet count this pod last read may limit the key",
case.name
);
assert_eq!(
limiter.cache.get("key").unwrap().local_pending,
case.events_counted_here_since + 1,
"{}: the guard withholds the limit and changes nothing else",
case.name
);
}
}
#[tokio::test]
async fn test_read_outage_engages_on_failed_reads_and_clears_on_the_next_success() {
let mut redis = MockRedisClient::new();
for call in 0..20 {
redis = redis.mget_error_at_call(call, CustomRedisError::Timeout);
}
let limiter = GlobalRateLimiterImpl::new(outage_config(10), vec![Arc::new(redis)]).unwrap();
let _ = limiter.check_limit("key", 10, None).await;
assert!(
limited(limiter.check_limit("key", 1, None).await),
"before any read has failed, this pod's own count limits the key"
);
wait_until!(
"failed reads must engage the guard",
!limited(limiter.check_limit("key", 10, None).await)
);
wait_until!(
"the first successful read must clear the guard, so this pod's count limits again",
limited(limiter.check_limit("key", 10, None).await)
);
}
#[tokio::test]
async fn test_a_pod_that_never_reads_never_enters_a_read_outage() {
let config = GlobalRateLimiterConfig {
max_pending_sync_entries: 0,
..outage_config(10)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![failing_reads()]).unwrap();
let _ = limiter.check_limit("key", 10, None).await;
// Ticks run with nothing to read. The claim holds whether zero or many ran.
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
limited(limiter.check_limit("key", 1, None).await),
"no read was attempted, so none failed, and this pod keeps limiting on its own count"
);
}
#[tokio::test]
async fn test_read_outage_is_judged_per_redis_instance() {
let clients: Vec<Arc<dyn Client + Send + Sync>> =
vec![failing_reads(), Arc::new(MockRedisClient::new())];
let limiter = GlobalRateLimiterImpl::new(outage_config(10_000), clients).unwrap();
// `instance_index` is the routing rule every pod shares; the test needs one key per instance.
let key_on = |instance: usize| -> String {
(0..)
.map(|i| format!("key{i}"))
.find(|k| instance_index(k, 2) == instance)
.unwrap()
};
let (on_down, on_healthy) = (key_on(0), key_on(1));
for key in [&on_down, &on_healthy] {
let _ = limiter.check_limit(key, 10_000, None).await;
}
wait_until!(
"the failing instance must enter a read outage",
!limited(limiter.check_limit(&on_down, 1, None).await)
);
assert!(
limited(limiter.check_limit(&on_healthy, 10_000, None).await),
"an outage on one instance must not withhold limits on a key the healthy one owns"
);
}
#[tokio::test]
async fn test_one_successful_read_in_a_tick_keeps_the_instance_healthy() {
let redis =
Arc::new(MockRedisClient::new().mget_error_at_call(0, CustomRedisError::Timeout));
let config = GlobalRateLimiterConfig {
// One entity per read command, so two keys make two reads in one tick.
max_keys_per_command: 2,
..outage_config(10_000)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![redis.clone()]).unwrap();
for key in ["a", "b"] {
let _ = limiter.check_limit(key, 10_000, None).await;
}
wait_until!(
"both reads must go out",
redis.get_calls().iter().filter(|c| c.op == "mget").count() >= 2
);
for key in ["a", "b"] {
assert!(
limited(limiter.check_limit(key, 10_000, None).await),
"one read failed and one succeeded, so the instance is reachable and {key} stays \
limited on this pod's count"
);
}
}
#[test]
fn test_read_health_keeps_the_start_of_the_current_run_of_failed_reads() {
let health = ReadHealth::new(2);
let at = |secs: u64| health.base + Duration::from_secs(secs);
assert_eq!(
health.failing_for(0, at(10)),
None,
"healthy until a read fails"
);
health.record_tick(0, false, at(10));
health.record_tick(0, false, at(40));
assert_eq!(
health.failing_for(0, at(70)),
Some(Duration::from_secs(60)),
"a later failed tick must not restart the run"
);
assert_eq!(
health.failing_for(1, at(70)),
None,
"each instance has its own run"
);
health.record_tick(0, true, at(80));
assert_eq!(
health.failing_for(0, at(90)),
None,
"one successful tick ends the run"
);
health.record_tick(0, false, at(100));
assert_eq!(
health.failing_for(0, at(105)),
Some(Duration::from_secs(5)),
"the next failure starts a new run"
);
}
#[test]
fn test_read_outage_limit_follows_the_window_unless_overridden() {
// (window_secs, override, expected_secs)
let cases = [
(60, None, Some(60)),
(600, None, Some(600)),
(120, Some(Duration::from_secs(30)), Some(30)),
(120, Some(Duration::ZERO), None),
];
for (window_secs, max_read_outage, expected_secs) in cases {
let config = GlobalRateLimiterConfig {
window_interval: Duration::from_secs(window_secs),
max_read_outage,
..test_config()
};
assert_eq!(
GlobalRateLimiterImpl::read_outage_limit(&config),
expected_secs.map(Duration::from_secs),
"window {window_secs}s, override {max_read_outage:?}: unset follows the window, zero disables the guard"
);
}
}
#[tokio::test]
async fn test_idle_tier_key_above_floor_still_syncs() {
let client = Arc::new(MockRedisClient::new());
let config = GlobalRateLimiterConfig {
global_threshold: 1000,
..config_with_floor(10)
};
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// Locally accumulated events don't decay, so this entry sits at level 50:
// idle by pressure (0.05 < 0.1) but well above the absolute floor, and
// last synced longer ago than the Low cadence (4 * 15s).
limiter.cache.insert(
"fleet_hot".to_string(),
CacheEntry {
estimated_count: 0.0,
synced_at: Some(Instant::now() - Duration::from_secs(120)),
local_pending: 50,
pressure: 0.05,
},
);
limiter.check_limit("fleet_hot", 1, None).await;
assert!(
limiter.pending_sync.contains("fleet_hot"),
"an idle-tier key above the floor must still sync, else a key hot across \
the fleet but cold on any single node is never discovered and never limited"
);
}
#[tokio::test]
async fn test_tick_bounds_drain_and_chunks_reads() {
let mock = Arc::new(MockRedisClient::new());
let client: Arc<dyn Client + Send + Sync> = mock.clone();
let config = GlobalRateLimiterConfig {
max_sync_keys_per_tick: 10,
// 2 entities per read command (two epoch keys each).
max_keys_per_command: 4,
..config_with_floor(0)
};
let cache: Cache<String, CacheEntry> = Cache::builder().max_capacity(1000).build();
let pending: Arc<DashSet<String>> = Arc::new(DashSet::new());
for i in 0..25 {
pending.insert(format!("k{i}"));
}
let mut writes: HashMap<(String, i64), u64> = HashMap::new();
GlobalRateLimiterImpl::tick(
&config,
std::slice::from_ref(&client),
&cache,
&pending,
&mut writes,
&ReadHealth::new(1),
"test",
1,
)
.await;
assert_eq!(
pending.len(),
15,
"tick must take at most max_sync_keys_per_tick and leave the remainder \
queued -- deferring keeps the tick inside its interval, dropping them \
would silently lose syncs"
);
let mget_calls = mock
.get_calls()
.into_iter()
.filter(|c| c.op == "mget")
.count();
assert_eq!(
mget_calls, 5,
"10 drained keys at 2 entities per command must issue 5 bounded MGETs, \
not one oversized command that cannot fit the per-command timeout"
);
}
#[tokio::test]
async fn test_sync_dedup() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// Stale entry: will trigger sync
limiter.cache.insert(
"dedup_key".to_string(),
CacheEntry {
estimated_count: 5.0,
synced_at: Some(Instant::now() - Duration::from_secs(60)),
local_pending: 0,
pressure: 0.5,
},
);
// Call twice, should only appear once in pending_sync
let _ = limiter.check_limit("dedup_key", 1, None).await;
let _ = limiter.check_limit("dedup_key", 1, None).await;
let count = limiter
.pending_sync
.iter()
.filter(|r| r.key() == "dedup_key")
.count();
assert_eq!(count, 1, "pending_sync should deduplicate entries");
}
// --- Redis client selection tests ---
#[test]
fn test_new_returns_error_for_empty_redis_instances() {
let config = test_config();
let result = GlobalRateLimiterImpl::new(config, vec![]);
assert!(result.is_err());
let err = match result {
Err(e) => e,
Ok(_) => panic!("Expected error for empty redis_instances"),
};
assert!(
err.to_string()
.contains("requires at least one Redis instance"),
"Error message should mention Redis instance requirement, got: {err}"
);
}
#[test]
fn test_select_redis_client_single_instance() {
let client = Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>;
let clients = vec![client];
let (_, idx) = select_redis_client("any_key", &clients);
assert_eq!(idx, 0, "Single instance should always return index 0");
let (_, idx) = select_redis_client("another_key", &clients);
assert_eq!(idx, 0, "Single instance should always return index 0");
}
#[test]
fn test_select_redis_client_consistent_mapping() {
let clients: Vec<Arc<dyn Client + Send + Sync>> = (0..3)
.map(|_| Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>)
.collect();
let key = "test_key_for_consistency";
let (_, idx1) = select_redis_client(key, &clients);
let (_, idx2) = select_redis_client(key, &clients);
let (_, idx3) = select_redis_client(key, &clients);
assert_eq!(idx1, idx2, "Same key should always map to same instance");
assert_eq!(idx2, idx3, "Same key should always map to same instance");
}
#[test]
fn test_select_redis_client_distributes_keys() {
let clients: Vec<Arc<dyn Client + Send + Sync>> = (0..3)
.map(|_| Arc::new(MockRedisClient::new()) as Arc<dyn Client + Send + Sync>)
.collect();
let mut indices = std::collections::HashSet::new();
for i in 0..100 {
let key = format!("key_{i}");
let (_, idx) = select_redis_client(&key, &clients);
indices.insert(idx);
}
assert!(
indices.len() > 1,
"Multiple keys should distribute across instances"
);
}
// --- Process read results tests ---
#[test]
fn test_process_read_results_updates_cache() {
let config = test_config();
let cache = Cache::builder()
.max_capacity(100)
.time_to_live(Duration::from_secs(60))
.time_to_idle(Duration::from_secs(30))
.build();
// Seed cache with an old entry
cache.insert(
"entity_a".to_string(),
CacheEntry {
estimated_count: 0.0,
synced_at: Some(Instant::now() - Duration::from_secs(30)),
local_pending: 3,
pressure: 0.0,
},
);
let sync_keys = vec!["entity_a".to_string()];
// Results: current_epoch=7, prev_epoch=3
let results: Vec<Option<Vec<u8>>> = vec![Some(b"7".to_vec()), Some(b"3".to_vec())];
let now = DateTime::from_timestamp(90, 0).unwrap(); // progress = 0.5 in 60s window
GlobalRateLimiterImpl::process_read_results(
&config, &cache, &sync_keys, &results, now, "test",
);
let entry = cache.get("entity_a").unwrap();
// weighted = 3 * 0.5 + 7 = 8.5
assert!(
(entry.estimated_count - 8.5).abs() < 0.01,
"estimated_count should be ~8.5, got {}",
entry.estimated_count
);
// pressure = 8.5 / 10 = 0.85
assert!(
(entry.pressure - 0.85).abs() < 0.01,
"pressure should be ~0.85, got {}",
entry.pressure
);
// local_pending reset to 0 on sync (avoids double-counting)
assert_eq!(entry.local_pending, 0);
}
#[test]
fn test_process_read_results_custom_key_pressure() {
let config = test_config();
set_custom_keys(&config, &[("custom_entity", 100)]);
let cache = Cache::builder()
.max_capacity(100)
.time_to_live(Duration::from_secs(60))
.time_to_idle(Duration::from_secs(30))
.build();
cache.insert(
"custom_entity".to_string(),
CacheEntry {
estimated_count: 0.0,
synced_at: Some(Instant::now() - Duration::from_secs(30)),
local_pending: 3,
pressure: 0.0,
},
);
let sync_keys = vec!["custom_entity".to_string()];
let results: Vec<Option<Vec<u8>>> = vec![Some(b"7".to_vec()), Some(b"3".to_vec())];
let now = DateTime::from_timestamp(90, 0).unwrap();
GlobalRateLimiterImpl::process_read_results(
&config, &cache, &sync_keys, &results, now, "test",
);
let entry = cache.get("custom_entity").unwrap();
assert!(
(entry.estimated_count - 8.5).abs() < 0.01,
"estimated_count should be ~8.5, got {}",
entry.estimated_count
);
// pressure = 8.5 / 100 (custom threshold) = 0.085
assert!(
(entry.pressure - 0.085).abs() < 0.01,
"pressure should be ~0.085 for custom threshold 100, got {}",
entry.pressure
);
assert_eq!(entry.local_pending, 0);
}
#[test]
fn test_process_read_results_zeroes_local_pending() {
let config = test_config();
let now = DateTime::from_timestamp(90, 0).unwrap();
for prior_pending in [0u64, 1, 5, 100, 10_000] {
let cache = Cache::builder()
.max_capacity(100)
.time_to_live(Duration::from_secs(60))
.time_to_idle(Duration::from_secs(30))
.build();
cache.insert(
"key".to_string(),
CacheEntry {
estimated_count: 0.0,
synced_at: Some(Instant::now() - Duration::from_secs(30)),
local_pending: prior_pending,
pressure: 0.0,
},
);
let sync_keys = vec!["key".to_string()];
let results: Vec<Option<Vec<u8>>> = vec![Some(b"5".to_vec()), Some(b"2".to_vec())];
GlobalRateLimiterImpl::process_read_results(
&config, &cache, &sync_keys, &results, now, "test",
);
let entry = cache.get("key").unwrap();
assert_eq!(
entry.local_pending, 0,
"local_pending should be 0 after sync regardless of prior value ({prior_pending})"
);
}
}
// --- Tier distribution scan tests ---
fn seed_entry(pressure: f64) -> CacheEntry {
CacheEntry {
estimated_count: 0.0,
synced_at: Some(Instant::now() - Duration::from_secs(30)),
local_pending: 0,
pressure,
}
}
#[test]
fn test_scan_tier_counts_tallies_distribution() {
let cache = Cache::builder().max_capacity(100).build();
cache.insert("a".to_string(), seed_entry(0.0)); // idle
cache.insert("b".to_string(), seed_entry(0.05)); // idle
cache.insert("c".to_string(), seed_entry(0.3)); // low
cache.insert("d".to_string(), seed_entry(0.6)); // normal
cache.insert("e".to_string(), seed_entry(0.9)); // hot
cache.insert("f".to_string(), seed_entry(0.95)); // hot
cache.run_pending_tasks();
let counts = GlobalRateLimiterImpl::scan_tier_counts(&cache);
assert_eq!(counts[PressureTier::Idle.index()], 2);
assert_eq!(counts[PressureTier::Low.index()], 1);
assert_eq!(counts[PressureTier::Normal.index()], 1);
assert_eq!(counts[PressureTier::Hot.index()], 2);
assert_eq!(counts.iter().sum::<u64>(), cache.entry_count());
}
#[test]
fn test_scan_tier_counts_excludes_evicted_entries() {
// Size-based eviction: only the surviving entries should be tallied,
// and the total must match entry_count after maintenance.
let cache = Cache::builder().max_capacity(3).build();
for i in 0..10 {
cache.insert(format!("k{i}"), seed_entry(0.9)); // all hot
}
cache.run_pending_tasks();
let counts = GlobalRateLimiterImpl::scan_tier_counts(&cache);
assert_eq!(counts.iter().sum::<u64>(), cache.entry_count());
assert!(cache.entry_count() <= 3);
}
#[test]
fn test_scan_tier_counts_empty_cache() {
let cache: Cache<String, CacheEntry> = Cache::builder().max_capacity(100).build();
cache.run_pending_tasks();
let counts = GlobalRateLimiterImpl::scan_tier_counts(&cache);
assert_eq!(counts, [0, 0, 0, 0]);
}
#[test]
fn test_parse_redis_count() {
assert_eq!(parse_redis_count(&Some(b"42".to_vec())), 42);
assert_eq!(parse_redis_count(&Some(b"0".to_vec())), 0);
assert_eq!(parse_redis_count(&None), 0);
assert_eq!(parse_redis_count(&Some(b"not_a_number".to_vec())), 0);
assert_eq!(parse_redis_count(&Some(vec![])), 0);
}
// --- Dynamic custom-key map tests (ArcSwap + resolver) ---
#[test]
fn test_resolve_custom_default_exact_match() {
let config = test_config();
set_custom_keys(&config, &[("tok:did", 5), ("tok2", 9)]);
assert_eq!(config.resolve_custom("tok:did"), Some(5));
assert_eq!(config.resolve_custom("tok2"), Some(9));
assert_eq!(config.resolve_custom("missing"), None);
// Default resolver is exact: a token prefix must not match a token:did entry.
assert_eq!(config.resolve_custom("tok"), None);
}
#[test]
fn test_resolve_custom_injected_resolver_hierarchical() {
let mut config = test_config();
// Hierarchical policy: exact key first, then the token prefix before ':'.
config.custom_key_resolver = Some(Arc::new(|key: &str, map: &HashMap<String, u64>| {
if let Some(v) = map.get(key) {
return Some(*v);
}
key.split_once(':')
.and_then(|(tok, _)| map.get(tok).copied())
}));
set_custom_keys(&config, &[("tok", 7), ("tok:vip", 100)]);
assert_eq!(config.resolve_custom("tok:vip"), Some(100)); // exact wins
assert_eq!(config.resolve_custom("tok:other"), Some(7)); // falls back to token
assert_eq!(config.resolve_custom("tok"), Some(7)); // token itself
assert_eq!(config.resolve_custom("nope:x"), None); // no match at any level
}
#[tokio::test]
async fn test_custom_keys_swap_visible_to_running_limiter() {
let client = Arc::new(MockRedisClient::new());
let config = test_config();
// The refresh task swaps through a clone of the config; hold one here to
// drive the same shared ArcSwap the running limiter reads.
let cfg_handle = config.clone();
let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert!(!limiter.is_custom_key("dyn_key"));
let result = limiter.check_custom_limit("dyn_key", 1, None).await;
assert!(matches!(result, EvalResult::NotApplicable));
cfg_handle
.custom_keys
.store(Arc::new(HashMap::from([("dyn_key".to_string(), 42u64)])));
assert!(limiter.is_custom_key("dyn_key"));
// Now subject to a custom limit: a fresh key is Allowed (cache miss), not NotApplicable.
let result = limiter.check_custom_limit("dyn_key", 1, None).await;
assert!(
matches!(result, EvalResult::Allowed),
"swapped-in custom key should be evaluated, got {result:?}"
);
}
#[test]
fn test_custom_keys_swap_shared_across_config_clones() {
// The background task holds a *clone* of the config; a swap through the
// clone must be visible to the original (shared ArcSwap).
let config = test_config();
let bg_clone = config.clone();
assert_eq!(config.resolve_custom("k"), None);
bg_clone
.custom_keys
.store(Arc::new(HashMap::from([("k".to_string(), 3u64)])));
assert_eq!(config.resolve_custom("k"), Some(3));
}
#[test]
fn test_custom_keys_concurrent_read_is_consistent() {
// Readers never observe a torn map: each load sees either the fully-old
// or fully-new map, never a missing/partial value.
use std::sync::atomic::{AtomicBool, Ordering};
let config = Arc::new(test_config());
set_custom_keys(&config, &[("k", 1)]);
let stop = Arc::new(AtomicBool::new(false));
let reader = {
let config = config.clone();
let stop = stop.clone();
std::thread::spawn(move || {
while !stop.load(Ordering::Relaxed) {
let v = config.resolve_custom("k");
assert!(matches!(v, Some(1) | Some(100)), "torn read: {v:?}");
}
})
};
for i in 0..10_000 {
let v = if i % 2 == 0 { 1u64 } else { 100u64 };
config
.custom_keys
.store(Arc::new(HashMap::from([("k".to_string(), v)])));
}
stop.store(true, Ordering::Relaxed);
reader.join().unwrap();
}
// --- Refresh path tests (refresh_custom_keys_once via a mock source) ---
#[tokio::test]
async fn test_refresh_once_applies_thresholds() {
let config = test_config();
set_custom_keys(&config, &[("seed", 1)]);
let source = MockCustomKeyThresholdSource::with_thresholds(Some(HashMap::from([(
"dyn".to_string(),
99u64,
)])));
GlobalRateLimiterImpl::refresh_custom_keys_once(&config, &source, "test").await;
// Fetched map replaces the seed wholesale (Redis is authoritative).
assert_eq!(config.resolve_custom("dyn"), Some(99));
assert_eq!(config.resolve_custom("seed"), None);
}
#[tokio::test]
async fn test_refresh_once_absent_key_is_fail_static() {
let config = test_config();
set_custom_keys(&config, &[("seed", 1)]);
let source = MockCustomKeyThresholdSource::with_thresholds(None);
GlobalRateLimiterImpl::refresh_custom_keys_once(&config, &source, "test").await;
// Absent key is ambiguous (fresh rollout / eviction / accidental delete),
// not an authoritative clear: the current thresholds must survive.
assert_eq!(config.resolve_custom("seed"), Some(1));
}
#[tokio::test]
async fn test_refresh_once_empty_blob_clears_thresholds() {
let config = test_config();
set_custom_keys(&config, &[("seed", 1)]);
// An explicit empty blob (`{}` in Redis) is the deliberate clear signal,
// distinct from an absent key: it replaces the map with an empty one.
let source = MockCustomKeyThresholdSource::with_thresholds(Some(HashMap::new()));
GlobalRateLimiterImpl::refresh_custom_keys_once(&config, &source, "test").await;
assert_eq!(config.resolve_custom("seed"), None);
}
#[tokio::test]
async fn test_refresh_once_error_is_fail_static() {
let config = test_config();
set_custom_keys(&config, &[("keep", 7)]);
let source =
MockCustomKeyThresholdSource::with_error(common_redis::CustomRedisError::Timeout);
GlobalRateLimiterImpl::refresh_custom_keys_once(&config, &source, "test").await;
// A failed fetch must not disturb the current thresholds.
assert_eq!(config.resolve_custom("keep"), Some(7));
}
// --- Refresh task lifecycle (spawn on new(), stop on shutdown()) ---
/// Poll `cond`, driving the paused clock forward so the interval-based refresh
/// task gets a chance to tick, up to a bounded number of iterations.
async fn wait_until<F: Fn() -> bool>(cond: F) -> bool {
for _ in 0..50 {
if cond() {
return true;
}
tokio::time::advance(Duration::from_secs(60)).await;
tokio::task::yield_now().await;
}
cond()
}
#[tokio::test(start_paused = true)]
async fn test_refresh_task_spawns_ticks_and_stops_on_shutdown() {
let mock = Arc::new(MockCustomKeyThresholdSource::with_thresholds(Some(
HashMap::from([("first".to_string(), 11u64)]),
)));
let mut config = test_config();
// Hold the shared ArcSwap so we can observe swaps the spawned task makes.
let observed = config.custom_keys.clone();
config.custom_key_source = Some(mock.clone() as Arc<dyn CustomKeyThresholdSource>);
config.custom_key_refresh_interval = Duration::from_secs(60);
let client = Arc::new(MockRedisClient::new());
let mut limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
// The task ticks immediately on spawn and applies the initial map.
assert!(
wait_until(|| observed.load().get("first") == Some(&11)).await,
"refresh task should apply the initial map after spawn"
);
// A subsequent Redis-side change is picked up on the next tick.
mock.set_thresholds(Some(HashMap::from([("second".to_string(), 22u64)])))
.await;
assert!(
wait_until(|| observed.load().get("second") == Some(&22)).await,
"refresh task should apply an updated map on a later tick"
);
// After shutdown the task must exit: let it observe the closed stop channel.
limiter.shutdown();
for _ in 0..20 {
tokio::task::yield_now().await;
}
// A change made after shutdown must never be applied.
mock.set_thresholds(Some(HashMap::from([("third".to_string(), 33u64)])))
.await;
for _ in 0..10 {
tokio::time::advance(Duration::from_secs(120)).await;
tokio::task::yield_now().await;
}
assert!(
observed.load().get("third").is_none(),
"refresh task must not apply changes after shutdown"
);
// The last pre-shutdown value is still in place.
assert_eq!(observed.load().get("second"), Some(&22));
}
#[tokio::test(start_paused = true)]
async fn test_zero_refresh_interval_is_clamped_not_panicking() {
// A zero interval would panic `tokio::time::interval` and kill the
// detached refresh task; the clamp must keep it running and ticking.
let mock = Arc::new(MockCustomKeyThresholdSource::with_thresholds(Some(
HashMap::from([("only".to_string(), 9u64)]),
)));
let mut config = test_config();
let observed = config.custom_keys.clone();
config.custom_key_source = Some(mock.clone() as Arc<dyn CustomKeyThresholdSource>);
config.custom_key_refresh_interval = Duration::ZERO;
let client = Arc::new(MockRedisClient::new());
let _limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap();
assert!(
wait_until(|| observed.load().get("only") == Some(&9)).await,
"clamped refresh task should still apply the initial map"
);
}
}