:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use crate::config::Config;
use arc_swap::ArcSwap;
use chrono::Utc;
use common_redis::Client;
use limiters::custom_key_source::{CustomKeyThresholdSource, RedisCustomKeyThresholdSource};
use limiters::global_rate_limiter::{
CustomKeyResolver, EvalResult, GlobalRateLimitResponse,
GlobalRateLimiter as CommonGlobalRateLimiter, GlobalRateLimiterConfig,
GlobalRateLimiterImpl as CommonGlobalRateLimiterImpl,
};
use metrics::counter;
use tracing::{error, info, warn};
#[cfg(test)]
use chrono::DateTime;
const MAX_DISTINCT_ID_CHARS: usize = 128;
fn truncate_str(s: &str, max_chars: usize) -> &str {
match s.char_indices().nth(max_chars) {
Some((idx, _)) => &s[..idx],
None => s,
}
}
pub enum GlobalRateLimitKey<'a> {
/// Token-only key. Not currently used in production call sites.
Token(&'a str),
TokenDistinctId(&'a str, &'a str),
}
impl<'a> GlobalRateLimitKey<'a> {
pub fn to_cache_key(&self) -> Cow<'a, str> {
match self {
Self::Token(t) => Cow::Borrowed(t),
Self::TokenDistinctId(t, d) => {
Cow::Owned(format!("{t}:{}", truncate_str(d, MAX_DISTINCT_ID_CHARS)))
}
}
}
}
/// Redis key prefix for the AI lane's byte budget. Unlike the event limiters
/// this deliberately carries no `capture_mode`: AI events reach the AI topic
/// from the capture-ai deployment and from AI traffic diverted on
/// capture-analytics, so one token must draw on one budget across both. Both
/// deployments therefore have to point at the same rate-limiter Redis.
const AI_BYTES_REDIS_KEY_PREFIX: &str = "@ph/grl/capture/ai_bytes";
/// The per-instance knobs that distinguish one capture limiter from another.
/// Named fields rather than positional arguments: several are same-typed (two
/// `&str`, two `bool`) and a swap would silently misconfigure a limiter.
struct LimiterSpec<'a> {
/// Sliding window this limiter counts over. Sets Redis epoch numbering, so
/// limiters that share a key prefix must agree on it.
window_secs: u64,
/// Env var that supplied `window_secs`, so a zero window names the knob the
/// operator has to fix rather than the limiter that noticed.
window_env_var: &'a str,
/// Default budget per window for keys with no custom override.
threshold: u64,
/// Static `key=value` CSV seeding the custom-key map.
custom_keys_csv: Option<&'a String>,
/// Multiplier applied to each seeded CSV value, for limiters whose
/// operator-facing unit is not the window budget: the AI byte limiter is
/// configured in bytes/second and enforces bytes/window. `1` when the CSV
/// already carries per-window values.
custom_key_scale: u64,
local_cache_max_entries: u64,
/// Minimum local level before a key earns a Redis sync. The unit is
/// whatever the limiter counts, so an event limiter's floor is in events
/// and the AI byte limiter's is in bytes.
min_sync_floor: u64,
redis_key_prefix: &'a str,
metrics_scope: &'a str,
/// Whether to wire the Redis-backed dynamic threshold source.
enable_dynamic_source: bool,
/// Evaluate and report, but never enforce.
dry_run: bool,
}
/// The window the AI byte budget is enforced over, and the env var that
/// supplied it. `AI_BYTE_LIMIT_WINDOW_INTERVAL_SECS` wins when set; otherwise
/// the shared limiter window applies. Every place that derives the AI byte
/// budget from a window must call this, so the enforced budget and any
/// diagnostic computed from it cannot disagree.
pub fn ai_byte_limit_window(config: &Config) -> (u64, &'static str) {
match config.ai_byte_limit_window_interval_secs {
Some(secs) => (secs, "AI_BYTE_LIMIT_WINDOW_INTERVAL_SECS"),
None => (
config.global_rate_limit_window_interval_secs,
"GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS",
),
}
}
/// Writer and optional replica URL of one limiter Redis cluster.
#[derive(Debug, PartialEq)]
struct RedisClusterUrls<'a> {
writer: &'a str,
reader: Option<&'a str>,
}
pub struct GlobalRateLimiter {
limiter: Box<dyn CommonGlobalRateLimiter>,
dry_run: bool,
}
impl GlobalRateLimiter {
/// Create a per-(token, distinct_id) rate limiter sharing the given Redis instances.
pub fn new_token_distinct_id(
config: &Config,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
) -> anyhow::Result<Self> {
let prefix = format!(
"@ph/grl/capture/tok_distid/{}",
config.capture_mode.as_tag()
);
let metrics_scope = format!("{}_tok_distid", config.capture_mode.as_tag());
Self::build(
config,
redis_instances,
LimiterSpec {
window_secs: config.global_rate_limit_window_interval_secs,
window_env_var: "GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS",
threshold: config.global_rate_limit_token_distinctid_threshold,
custom_keys_csv: config
.global_rate_limit_token_distinctid_overrides_csv
.as_ref(),
custom_key_scale: 1,
local_cache_max_entries: config
.global_rate_limit_token_distinctid_local_cache_max_entries,
min_sync_floor: config.global_rate_limit_min_sync_floor,
redis_key_prefix: &prefix,
metrics_scope: &metrics_scope,
enable_dynamic_source: config.global_rate_limit_custom_threshold_key.is_some(),
dry_run: config.global_rate_limit_dry_run,
},
)
}
/// Create a per-token rate limiter sharing the given Redis instances.
/// Not currently wired into production call sites -- retained for future use.
pub fn new_token(
config: &Config,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
) -> anyhow::Result<Self> {
let prefix = format!("@ph/grl/capture/token/{}", config.capture_mode.as_tag());
let metrics_scope = format!("{}_token", config.capture_mode.as_tag());
Self::build(
config,
redis_instances,
LimiterSpec {
window_secs: config.global_rate_limit_window_interval_secs,
window_env_var: "GLOBAL_RATE_LIMIT_WINDOW_INTERVAL_SECS",
threshold: config.global_rate_limit_token_threshold,
custom_keys_csv: config.global_rate_limit_token_overrides_csv.as_ref(),
custom_key_scale: 1,
local_cache_max_entries: config.global_rate_limit_token_local_cache_max_entries,
min_sync_floor: config.global_rate_limit_min_sync_floor,
redis_key_prefix: &prefix,
metrics_scope: &metrics_scope,
// No dynamic source for the token-only limiter; its resolver's token
// fallback never fires for bare token keys.
enable_dynamic_source: false,
dry_run: config.global_rate_limit_dry_run,
},
)
}
/// Create the AI lane's per-token byte budget, sharing the given Redis instances.
///
/// Nothing about the underlying limiter changes: the counts it accumulates
/// are bytes rather than events, so the same sliding window that caps
/// events/window elsewhere caps bytes/window here. Callers pass an event's
/// serialized size as the count.
///
/// Budgets are configured in bytes/second and scaled to the AI byte window
/// (see `ai_byte_limit_window`), so the knob keeps its meaning whatever that
/// window is set to.
pub fn new_ai_bytes(
config: &Config,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
) -> anyhow::Result<Self> {
// `build` refuses to boot on a zero window, so this scaling never makes
// the budget zero.
let (window_secs, window_env_var) = ai_byte_limit_window(config);
let metrics_scope = format!("{}_ai_bytes", config.capture_mode.as_tag());
Self::build(
config,
redis_instances,
LimiterSpec {
window_secs,
window_env_var,
threshold: config.ai_byte_limit_per_second.saturating_mul(window_secs),
custom_keys_csv: config.ai_byte_limit_overrides_csv.as_ref(),
custom_key_scale: window_secs,
local_cache_max_entries: config.ai_byte_limit_local_cache_max_entries,
// No floor. The shared knob is an event count, and this
// limiter counts bytes, so inheriting it would be a unit
// confusion that reads as deliberate. The floor exists to keep
// an unbounded key space off Redis; this limiter's keys are
// projects sending AI traffic, few enough that syncing all of
// them is cheap and accurate enforcement is worth more.
min_sync_floor: 0,
redis_key_prefix: AI_BYTES_REDIS_KEY_PREFIX,
metrics_scope: &metrics_scope,
// Thresholds come from config only; the Django-written blob
// holds event-count limits, in different units.
enable_dynamic_source: false,
dry_run: config.ai_byte_limit_dry_run,
},
)
}
/// Hierarchical custom-key resolver. Always applied to capture limiters,
/// whether thresholds come from the static CSV seed or the dynamic source.
///
/// A lookup key is either `token` or `token:distinct_id` (the limiter's cache
/// key). Resolution tries the exact key first, then falls back to the token
/// prefix (everything before the first `:`) so a token-level override applies
/// to all of that token's `token:distinct_id` keys. Keeps capture's key
/// structure out of the common crate.
fn hierarchical_resolver() -> CustomKeyResolver {
Arc::new(|key: &str, map: &HashMap<String, u64>| {
if let Some(v) = map.get(key) {
return Some(*v);
}
key.split_once(':')
.and_then(|(token, _)| map.get(token).copied())
})
}
fn build(
config: &Config,
redis_instances: Vec<Arc<dyn Client + Send + Sync>>,
spec: LimiterSpec<'_>,
) -> anyhow::Result<Self> {
// A zero window panics the background task on its first tick, so no read ever
// lands and pods limit on local counts alone. Refuse to boot instead.
if spec.window_secs == 0 {
anyhow::bail!(
"invalid configuration: {} must be greater than 0 (limiter {}); \
a zero window gives every key an infinite leak rate, so no limit is ever enforced",
spec.window_env_var,
spec.metrics_scope
);
}
// Seed the (swappable) custom-key map from the static CSV overrides. When a
// dynamic source is enabled, the common refresh loop replaces this map only
// from an explicit Redis blob; an absent key or an unreachable Redis is
// fail-static, so the CSV seed keeps applying until a blob is written.
let mut seed = Self::format_custom_keys(spec.custom_keys_csv);
if spec.custom_key_scale != 1 {
for value in seed.values_mut() {
*value = value.saturating_mul(spec.custom_key_scale);
}
}
// Build the dynamic source when enabled and its Redis key + URL are set.
// The source reads the JSON blob from the event-restrictions Redis (a
// separate store from the traffic-count Redis in `redis_instances`) and
// owns its own connection/reconnect; the common limiter runs the loop.
let custom_key_source: Option<Arc<dyn CustomKeyThresholdSource>> = if spec
.enable_dynamic_source
{
match (
config.global_rate_limit_custom_threshold_key.as_ref(),
config.event_restrictions_redis_url.as_ref(),
) {
(Some(key), Some(redis_url)) => {
let response_timeout = (config.redis_response_timeout_ms != 0)
.then(|| Duration::from_millis(config.redis_response_timeout_ms));
let connection_timeout = (config.redis_connection_timeout_ms != 0)
.then(|| Duration::from_millis(config.redis_connection_timeout_ms));
Some(Arc::new(RedisCustomKeyThresholdSource::new(
redis_url.clone(),
key.clone(),
response_timeout,
connection_timeout,
)))
}
_ => {
warn!(
"Dynamic custom thresholds requested but threshold key or Redis URL is unset; using static CSV seed"
);
None
}
}
} else {
None
};
let grl_config = GlobalRateLimiterConfig {
global_threshold: spec.threshold,
window_interval: Duration::from_secs(spec.window_secs),
sync_interval: Duration::from_secs(config.global_rate_limit_sync_interval_secs),
tick_interval: Duration::from_millis(config.global_rate_limit_tick_interval_ms),
redis_key_prefix: spec.redis_key_prefix.to_string(),
custom_keys: Arc::new(ArcSwap::from_pointee(seed)),
// Capture keys are always `token` or `token:distinct_id`, so the
// hierarchical resolver is always the correct policy — a token-level
// override applies to all of that token's `token:distinct_id` keys,
// for both the CSV seed and the dynamic map.
custom_key_resolver: Some(Self::hierarchical_resolver()),
custom_key_source,
custom_key_refresh_interval: Duration::from_secs(
config.global_rate_limit_custom_threshold_refresh_secs,
),
local_cache_max_entries: spec.local_cache_max_entries,
metrics_scope: spec.metrics_scope.to_string(),
min_sync_floor: spec.min_sync_floor,
max_read_outage: config
.global_rate_limit_max_read_outage_secs
.map(Duration::from_secs),
max_sync_keys_per_tick: config.global_rate_limit_max_sync_keys_per_tick,
max_keys_per_command: config.global_rate_limit_max_keys_per_command,
max_concurrent_commands: config.global_rate_limit_max_concurrent_commands,
max_write_batch_entries: config.global_rate_limit_max_write_batch_entries,
max_pending_sync_entries: config.global_rate_limit_max_pending_sync_entries,
global_read_timeout: Duration::from_millis(config.global_rate_limit_read_timeout_ms),
global_write_timeout: Duration::from_millis(config.global_rate_limit_write_timeout_ms),
local_cache_ttl: Duration::from_secs(config.global_rate_limit_local_cache_ttl_secs),
local_cache_idle_timeout: Duration::from_secs(
config.global_rate_limit_local_cache_idle_timeout_secs,
),
..Default::default()
};
let dry_run = spec.dry_run;
let limiter = match CommonGlobalRateLimiterImpl::new(grl_config, redis_instances) {
Ok(l) => l,
Err(e) => {
error!(error = %e, "Failed to initialize GlobalRateLimiter");
return Err(e);
}
};
if dry_run {
info!(
scope = spec.metrics_scope,
"GlobalRateLimiter initialized in dry-run mode (evaluating but not enforcing)"
);
}
Ok(Self {
limiter: Box::new(limiter),
dry_run,
})
}
/// Check if a key is rate limited. The key is resolved against the custom-key
/// map exactly once: `check_custom_limit` returns `NotApplicable` for keys
/// without a custom override, and only then do we fall back to the global
/// threshold check. Routing and threshold therefore come from a single map
/// snapshot, so a concurrent custom-map swap can't misroute a request (a
/// prior `is_custom_key` probe + separate check made two independent loads
/// that could straddle a swap). Exactly one enforcing check fires — the
/// `NotApplicable` probe returns before touching the cache or Redis batch
/// channel, so there's no double-enqueue.
///
/// In dry-run mode the underlying limiter is still evaluated (counts are
/// tracked, Redis is synced) but the result is suppressed: metrics and a
/// warn log are emitted, then `None` is returned so callers never enforce.
pub async fn is_limited(&self, key: &str, count: u64) -> Option<GlobalRateLimitResponse> {
let result = match self
.limiter
.check_custom_limit(key, count, Some(Utc::now()))
.await
{
// No custom override for this key: enforce the global threshold.
EvalResult::NotApplicable => self.is_global_key_limited(key, count).await,
EvalResult::Limited(response) => Some(response),
// Allowed on a key that HAS a custom override: not limited,
// and we must not re-check it against the global threshold.
_ => None,
};
match (result, self.dry_run) {
(Some(response), true) => {
counter!(
"capture_global_rate_limiter_dry_run",
"key_type" => if response.is_custom_limited { "custom" } else { "global" },
)
.increment(1);
warn!(
key = key,
current_count = response.current_count,
threshold = response.threshold,
is_custom_limited = response.is_custom_limited,
dry_run = true,
"global rate limiter would have limited (dry run)"
);
None
}
(result, _) => result,
}
}
async fn is_global_key_limited(
&self,
key: &str,
count: u64,
) -> Option<GlobalRateLimitResponse> {
match self.limiter.check_limit(key, count, Some(Utc::now())).await {
EvalResult::Limited(response) => Some(response),
_ => None,
}
}
// trigger shutdown and stop pushing updates to global cache. Also stops the
// common custom-key refresh loop, if one was spawned.
pub fn shutdown(&mut self) {
self.limiter.shutdown();
}
/// Returns true if the key resolves to a custom threshold (exact match, or
/// token-prefix fallback via the hierarchical resolver).
pub fn is_custom_key(&self, key: &str) -> bool {
self.limiter.is_custom_key(key)
}
/// Clients for the per-(token, distinct_id) limiter, one per shard in shard order. Without
/// `GLOBAL_RATE_LIMIT_REDIS_URL` the limiter shares the main client.
pub async fn build_redis_shards(
config: &Config,
shared_redis: Arc<dyn Client + Send + Sync>,
) -> anyhow::Result<Vec<Arc<dyn Client + Send + Sync>>> {
let clusters = Self::redis_shard_urls(config)?;
if clusters.is_empty() {
return Ok(vec![shared_redis]);
}
if clusters.len() > 1 {
info!(
shards = clusters.len(),
"Global rate limiter sharding keys across Redis clusters"
);
}
let mut clients = Vec::with_capacity(clusters.len());
for cluster in clusters {
clients.push(Self::connect_redis(config, cluster).await?);
}
Ok(clients)
}
/// Client for a limiter that never shards (the AI byte limiter): the first cluster, or the
/// main client without `GLOBAL_RATE_LIMIT_REDIS_URL`.
pub async fn build_redis_client(
config: &Config,
shared_redis: Arc<dyn Client + Send + Sync>,
) -> anyhow::Result<Arc<dyn Client + Send + Sync>> {
match Self::first_cluster_urls(config) {
Some(cluster) => Self::connect_redis(config, cluster).await,
None => Ok(shared_redis),
}
}
fn first_cluster_urls(config: &Config) -> Option<RedisClusterUrls<'_>> {
config
.global_rate_limit_redis_url
.as_deref()
.map(|writer| RedisClusterUrls {
writer,
reader: config.global_rate_limit_redis_reader_url.as_deref(),
})
}
/// The order of the returned clusters is the routing table, so every pod must list them the same way.
fn redis_shard_urls(config: &Config) -> anyhow::Result<Vec<RedisClusterUrls<'_>>> {
let first = Self::first_cluster_urls(config);
match config.global_rate_limit_redis_shards {
1 => Ok(first.into_iter().collect()),
2 => {
let Some(first) = first else {
anyhow::bail!(
"GLOBAL_RATE_LIMIT_REDIS_SHARDS=2 requires GLOBAL_RATE_LIMIT_REDIS_URL"
);
};
let Some(writer) = config.global_rate_limit_redis_shard_1_url.as_deref() else {
anyhow::bail!(
"GLOBAL_RATE_LIMIT_REDIS_SHARDS=2 requires GLOBAL_RATE_LIMIT_REDIS_SHARD_1_URL"
);
};
Ok(vec![
first,
RedisClusterUrls {
writer,
reader: config.global_rate_limit_redis_shard_1_reader_url.as_deref(),
},
])
}
n => anyhow::bail!("GLOBAL_RATE_LIMIT_REDIS_SHARDS must be 1 or 2, got {n}"),
}
}
async fn connect_redis(
config: &Config,
cluster: RedisClusterUrls<'_>,
) -> anyhow::Result<Arc<dyn Client + Send + Sync>> {
let response_timeout = config
.global_rate_limit_redis_response_timeout_ms
.unwrap_or(config.redis_response_timeout_ms);
let connection_timeout = config
.global_rate_limit_redis_connection_timeout_ms
.unwrap_or(config.redis_connection_timeout_ms);
let response_timeout = if response_timeout == 0 {
None
} else {
Some(Duration::from_millis(response_timeout))
};
let connection_timeout = if connection_timeout == 0 {
None
} else {
Some(Duration::from_millis(connection_timeout))
};
if let Some(reader_url) = cluster.reader {
info!("Global rate limiter using read/write split Redis client");
let rw_config = common_redis::ReadWriteClientConfig::new(
cluster.writer.to_string(),
reader_url.to_string(),
common_redis::CompressionConfig::disabled(),
common_redis::RedisValueFormat::default(),
response_timeout,
connection_timeout,
);
Ok(Arc::new(rw_config.build().await?))
} else {
Ok(Arc::new(
common_redis::RedisClient::with_config(
cluster.writer.to_string(),
common_redis::CompressionConfig::disabled(),
common_redis::RedisValueFormat::default(),
response_timeout,
connection_timeout,
)
.await?,
))
}
}
#[cfg(test)]
pub(crate) fn new_with(limiter: impl CommonGlobalRateLimiter + 'static) -> Self {
Self {
limiter: Box::new(limiter),
dry_run: false,
}
}
/// Test helper: build a real limiter with the hierarchical resolver and its
/// custom-key map seeded from `csv` (no dynamic source), backed by a mock
/// Redis client. Exercises the static-CSV path with hierarchical resolution.
#[cfg(test)]
pub(crate) fn for_test_hierarchical_seeded(csv: Option<&str>) -> Self {
let csv_owned = csv.map(|s| s.to_string());
let grl_config = GlobalRateLimiterConfig {
global_threshold: 300_000,
redis_key_prefix: "test:grl".to_string(),
custom_keys: Arc::new(ArcSwap::from_pointee(Self::format_custom_keys(
csv_owned.as_ref(),
))),
custom_key_resolver: Some(Self::hierarchical_resolver()),
metrics_scope: "test".to_string(),
..Default::default()
};
let limiter = CommonGlobalRateLimiterImpl::new(
grl_config,
vec![Arc::new(common_redis::MockRedisClient::new())],
)
.expect("failed to build test limiter");
Self {
limiter: Box::new(limiter),
dry_run: false,
}
}
#[cfg(test)]
pub(crate) fn new_with_dry_run(limiter: impl CommonGlobalRateLimiter + 'static) -> Self {
Self {
limiter: Box::new(limiter),
dry_run: true,
}
}
/// Test helper: build a `GlobalRateLimiter` that returns `Limited` for any
/// key in `limited_keys` and `Allowed` otherwise. Custom limits always
/// return `NotApplicable`. Shared across capture pipeline tests that need
/// to simulate per-key global rate limiting.
#[cfg(test)]
pub(crate) fn mock_limiting(limited_keys: &[&str]) -> Self {
use async_trait::async_trait;
use std::collections::HashSet;
struct MockLimitingLimiter {
limited_keys: HashSet<String>,
}
#[async_trait]
impl CommonGlobalRateLimiter for MockLimitingLimiter {
async fn check_limit(
&self,
key: &str,
_count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
if self.limited_keys.contains(key) {
EvalResult::Limited(GlobalRateLimitResponse {
key: key.to_string(),
current_count: 100.0,
threshold: 10,
window_interval: std::time::Duration::from_secs(60),
sync_interval: std::time::Duration::from_secs(15),
is_custom_limited: false,
})
} else {
EvalResult::Allowed
}
}
async fn check_custom_limit(
&self,
_key: &str,
_count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
EvalResult::NotApplicable
}
fn is_custom_key(&self, _key: &str) -> bool {
false
}
fn shutdown(&mut self) {}
}
let limited_keys: HashSet<String> = limited_keys.iter().map(|k| k.to_string()).collect();
Self::new_with(MockLimitingLimiter { limited_keys })
}
/// Test helper: build a `GlobalRateLimiter` that tallies the counts charged
/// per key and limits once a key's running total reaches `threshold`, with
/// the charge for the current call included. Lets weight-charging call sites
/// (the AI byte budget) be tested without Redis.
///
/// Accounting is strict: a key is limited as soon as its total crosses the
/// threshold, including on its very first charge. Tests then say what they
/// mean — an event drops because of arithmetic a reader can check, not
/// because of a seam-specific exemption.
///
/// The real limiter also admits a key whose cache entry is missing, which
/// recurs on every eviction and every new pod.
/// [`Self::mock_budget_admitting_cold_keys`] models that, and one test pins
/// it rather than every test inheriting it.
///
/// Neither models the local cache's leaky-bucket decay, so a key never
/// recovers here.
#[cfg(test)]
pub(crate) fn mock_budget(threshold: u64) -> Self {
Self::mock_budget_with(threshold, false)
}
/// Test helper: [`Self::mock_budget`], plus the real limiter's fail-open on
/// a cache miss — a key's first charge is admitted however far over the
/// threshold it is.
#[cfg(test)]
pub(crate) fn mock_budget_admitting_cold_keys(threshold: u64) -> Self {
Self::mock_budget_with(threshold, true)
}
#[cfg(test)]
fn mock_budget_with(threshold: u64, admit_cold_keys: bool) -> Self {
use async_trait::async_trait;
use std::sync::Mutex;
struct MockBudgetLimiter {
threshold: u64,
admit_cold_keys: bool,
charged: Mutex<HashMap<String, u64>>,
}
#[async_trait]
impl CommonGlobalRateLimiter for MockBudgetLimiter {
async fn check_limit(
&self,
key: &str,
count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
let mut charged = self.charged.lock().unwrap();
let cold_key = !charged.contains_key(key);
let total = charged.entry(key.to_string()).or_insert(0);
*total += count;
let exempt = cold_key && self.admit_cold_keys;
if !exempt && *total >= self.threshold {
EvalResult::Limited(GlobalRateLimitResponse {
key: key.to_string(),
current_count: *total as f64,
threshold: self.threshold,
window_interval: std::time::Duration::from_secs(60),
sync_interval: std::time::Duration::from_secs(15),
is_custom_limited: false,
})
} else {
EvalResult::Allowed
}
}
async fn check_custom_limit(
&self,
_key: &str,
_count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
EvalResult::NotApplicable
}
fn is_custom_key(&self, _key: &str) -> bool {
false
}
fn shutdown(&mut self) {}
}
Self::new_with(MockBudgetLimiter {
threshold,
admit_cold_keys,
charged: Mutex::new(HashMap::new()),
})
}
// In capture deploys, the custom keys and rate limit thresholds should be
// supplied as a CSV list of <string_key>=<u64_value> elements. malformed
// elements will be logged and skipped during limiter initialization
fn format_custom_keys(custom_keys_csv: Option<&String>) -> HashMap<String, u64> {
let mut custom_keys = HashMap::new();
if custom_keys_csv.is_none() || custom_keys_csv.unwrap().is_empty() {
return custom_keys;
}
for elem in custom_keys_csv.unwrap().split(',') {
let (key, limit) = match elem.split_once('=') {
Some((k, l)) => (k, l),
None => {
error!(
input = elem,
"Global rate limiter: failed to split custom key-value pair"
);
continue;
}
};
let key = key.trim();
match limit.trim().parse::<u64>() {
Ok(value) => custom_keys.insert(key.to_string(), value),
Err(e) => {
error!(
error = e.to_string(),
key, limit, "Global rate limiter: failed to parse custom key value"
);
continue;
}
};
}
custom_keys
}
}
#[cfg(test)]
mod tests {
use super::*;
use async_trait::async_trait;
use rstest::rstest;
use std::collections::HashSet;
use std::sync::Mutex;
use std::time::Duration;
fn shard_config(extra: &[(&str, &str)]) -> Config {
let env: HashMap<String, String> = [
("REDIS_URL", "redis://localhost:6379/"),
("CAPTURE_MODE", "events"),
(
"CAPTURE_OUTPUT_ANALYTICS_MAIN_TOPIC",
"events_plugin_ingestion",
),
]
.iter()
.chain(extra.iter())
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
envconfig::Envconfig::init_from_hashmap(&env).expect("test config")
}
#[rstest]
#[case::no_dedicated_redis(&[], &[])]
#[case::shard_1_ignored_until_enabled(
&[
("GLOBAL_RATE_LIMIT_REDIS_URL", "redis://a"),
("GLOBAL_RATE_LIMIT_REDIS_SHARD_1_URL", "redis://b"),
],
&[("redis://a", None)],
)]
#[case::two_shards_in_order(
&[
("GLOBAL_RATE_LIMIT_REDIS_SHARDS", "2"),
("GLOBAL_RATE_LIMIT_REDIS_URL", "redis://a"),
("GLOBAL_RATE_LIMIT_REDIS_READER_URL", "redis://a-replica"),
("GLOBAL_RATE_LIMIT_REDIS_SHARD_1_URL", "redis://b"),
("GLOBAL_RATE_LIMIT_REDIS_SHARD_1_READER_URL", "redis://b-replica"),
],
&[("redis://a", Some("redis://a-replica")), ("redis://b", Some("redis://b-replica"))],
)]
fn redis_shard_urls_follow_the_shard_count(
#[case] env: &[(&str, &str)],
#[case] want: &[(&str, Option<&str>)],
) {
let config = shard_config(env);
let got: Vec<(&str, Option<&str>)> = GlobalRateLimiter::redis_shard_urls(&config)
.expect("valid shard config")
.into_iter()
.map(|c| (c.writer, c.reader))
.collect();
assert_eq!(got, want);
}
#[rstest]
#[case::shard_1_url_missing(
&[("GLOBAL_RATE_LIMIT_REDIS_SHARDS", "2"), ("GLOBAL_RATE_LIMIT_REDIS_URL", "redis://a")],
"GLOBAL_RATE_LIMIT_REDIS_SHARD_1_URL",
)]
#[case::first_url_missing(
&[("GLOBAL_RATE_LIMIT_REDIS_SHARDS", "2"), ("GLOBAL_RATE_LIMIT_REDIS_SHARD_1_URL", "redis://b")],
"requires GLOBAL_RATE_LIMIT_REDIS_URL",
)]
#[case::unsupported_count(&[("GLOBAL_RATE_LIMIT_REDIS_SHARDS", "3")], "must be 1 or 2")]
fn redis_shard_urls_reject_an_incomplete_shard_config(
#[case] env: &[(&str, &str)],
#[case] expected: &str,
) {
let err = GlobalRateLimiter::redis_shard_urls(&shard_config(env))
.expect_err("invalid shard config must fail startup");
assert!(
err.to_string().contains(expected),
"expected {expected:?} in {err}"
);
}
type CallLog = Arc<Mutex<Vec<&'static str>>>;
struct MockLimiter {
custom_keys: HashSet<String>,
check_limit_result: EvalResult,
check_custom_limit_result: EvalResult,
calls: CallLog,
}
impl MockLimiter {
fn new(
custom_keys: HashSet<String>,
check_limit_result: EvalResult,
check_custom_limit_result: EvalResult,
) -> (Self, CallLog) {
let calls: CallLog = Arc::new(Mutex::new(Vec::new()));
let mock = Self {
custom_keys,
check_limit_result,
check_custom_limit_result,
calls: calls.clone(),
};
(mock, calls)
}
}
#[async_trait]
impl CommonGlobalRateLimiter for MockLimiter {
async fn check_limit(
&self,
_key: &str,
_count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
self.calls.lock().unwrap().push("check_limit");
self.check_limit_result.clone()
}
async fn check_custom_limit(
&self,
_key: &str,
_count: u64,
_timestamp: Option<DateTime<Utc>>,
) -> EvalResult {
self.calls.lock().unwrap().push("check_custom_limit");
self.check_custom_limit_result.clone()
}
fn is_custom_key(&self, key: &str) -> bool {
self.calls.lock().unwrap().push("is_custom_key");
self.custom_keys.contains(key)
}
fn shutdown(&mut self) {}
}
fn make_limited_response(is_custom: bool) -> EvalResult {
EvalResult::Limited(GlobalRateLimitResponse {
key: "test".to_string(),
current_count: 100.0,
threshold: 10,
window_interval: Duration::from_secs(60),
sync_interval: Duration::from_secs(15),
is_custom_limited: is_custom,
})
}
#[tokio::test]
async fn test_is_limited_routes_to_global_for_unknown_key() {
let (mock, calls) = MockLimiter::new(
HashSet::new(),
make_limited_response(false),
EvalResult::NotApplicable,
);
let wrapper = GlobalRateLimiter::new_with(mock);
let result = wrapper.is_limited("unknown_key", 1).await;
assert!(result.is_some());
assert!(!result.unwrap().is_custom_limited);
assert_eq!(
*calls.lock().unwrap(),
vec!["check_custom_limit", "check_limit"],
"no custom override (NotApplicable) must fall back to the global check"
);
}
#[tokio::test]
async fn test_is_limited_routes_to_custom_for_registered_key() {
let (mock, calls) = MockLimiter::new(
HashSet::from(["registered".to_string()]),
EvalResult::Allowed,
make_limited_response(true),
);
let wrapper = GlobalRateLimiter::new_with(mock);
let result = wrapper.is_limited("registered", 1).await;
assert!(result.is_some());
assert!(result.unwrap().is_custom_limited);
assert_eq!(
*calls.lock().unwrap(),
vec!["check_custom_limit"],
"a custom Limited result must not trigger a second global check"
);
}
#[tokio::test]
async fn test_is_limited_custom_key_allowed_does_not_fall_through_to_global() {
let (mock, calls) = MockLimiter::new(
HashSet::from(["custom".to_string()]),
EvalResult::Allowed,
EvalResult::Allowed,
);
let wrapper = GlobalRateLimiter::new_with(mock);
let result = wrapper.is_limited("custom", 1).await;
assert!(result.is_none());
assert_eq!(
*calls.lock().unwrap(),
vec!["check_custom_limit"],
"an Allowed custom key must NOT fall through to the global check"
);
}
#[test]
fn test_global_rate_limit_key_to_cache_key() {
let cases: Vec<(GlobalRateLimitKey, &str, bool)> = vec![
(GlobalRateLimitKey::Token("abc"), "abc", true),
(GlobalRateLimitKey::Token(""), "", true),
(
GlobalRateLimitKey::TokenDistinctId("abc", "xyz"),
"abc:xyz",
false,
),
(
GlobalRateLimitKey::TokenDistinctId("abc", ""),
"abc:",
false,
),
];
for (key, expected, expect_borrowed) in cases {
let result = key.to_cache_key();
assert_eq!(&*result, expected, "key={expected}");
assert_eq!(
matches!(result, Cow::Borrowed(_)),
expect_borrowed,
"key={expected}: expected borrowed={expect_borrowed}"
);
}
}
#[test]
fn test_truncate_str() {
let short = "hello";
let exactly_128 = "a".repeat(128);
let over_128 = "b".repeat(200);
let truncated_128 = "b".repeat(128);
// Multi-byte: é is 2 bytes in UTF-8
let multibyte_at_boundary = format!("{}{}", "x".repeat(127), "é");
let cases: Vec<(&str, usize, &str)> = vec![
("", 128, ""),
(short, 128, short),
(&exactly_128, 128, &exactly_128),
(&over_128, 128, &truncated_128),
(&multibyte_at_boundary, 128, &multibyte_at_boundary),
];
for (input, max, expected) in &cases {
let result = truncate_str(input, *max);
assert_eq!(
result,
*expected,
"truncate_str({:?}, {max}) = {:?}, expected {:?}",
&input[..input.len().min(20)],
result,
expected
);
}
}
#[test]
fn test_token_distinct_id_cache_key_truncates_long_distinct_id() {
let token = "phc_abc";
let long_id = "d".repeat(300);
let key = GlobalRateLimitKey::TokenDistinctId(token, &long_id);
let result = key.to_cache_key();
let expected = format!("{token}:{}", &"d".repeat(MAX_DISTINCT_ID_CHARS));
assert_eq!(&*result, &expected);
assert_eq!(
result.len(),
token.len() + 1 + MAX_DISTINCT_ID_CHARS,
"cache key should be token + ':' + 128 chars"
);
}
#[test]
fn test_token_distinct_id_cache_key_preserves_short_distinct_id() {
let key = GlobalRateLimitKey::TokenDistinctId("tok", "short");
assert_eq!(&*key.to_cache_key(), "tok:short");
}
#[test]
fn test_token_distinct_id_truncation_utf8_safe() {
// 127 ASCII chars + a 2-byte UTF-8 char = 128 chars, 129 bytes
let prefix = "x".repeat(127);
let distinct_id = format!("{}é", prefix);
assert_eq!(distinct_id.chars().count(), 128);
let key = GlobalRateLimitKey::TokenDistinctId("t", &distinct_id);
let result = key.to_cache_key();
// Should keep all 128 chars (not split the é)
assert_eq!(&*result, &format!("t:{distinct_id}"));
// Now 128 ASCII + é = 129 chars, should truncate to 128
let prefix_129 = "x".repeat(128);
let distinct_id_129 = format!("{}é", prefix_129);
assert_eq!(distinct_id_129.chars().count(), 129);
let key2 = GlobalRateLimitKey::TokenDistinctId("t", &distinct_id_129);
let result2 = key2.to_cache_key();
assert_eq!(&*result2, &format!("t:{prefix_129}"));
}
// --- dry-run mode tests ---
#[tokio::test]
async fn test_dry_run_global_limited_returns_none() {
let (mock, calls) = MockLimiter::new(
HashSet::new(),
make_limited_response(false),
EvalResult::NotApplicable,
);
let wrapper = GlobalRateLimiter::new_with_dry_run(mock);
let result = wrapper.is_limited("some_key", 1).await;
assert!(
result.is_none(),
"dry-run should suppress Limited → None, got {result:?}"
);
assert_eq!(
*calls.lock().unwrap(),
vec!["check_custom_limit", "check_limit"],
"underlying limiter must still be called in dry-run mode"
);
}
#[tokio::test]
async fn test_dry_run_custom_limited_returns_none() {
let (mock, calls) = MockLimiter::new(
HashSet::from(["custom_key".to_string()]),
EvalResult::Allowed,
make_limited_response(true),
);
let wrapper = GlobalRateLimiter::new_with_dry_run(mock);
let result = wrapper.is_limited("custom_key", 1).await;
assert!(
result.is_none(),
"dry-run should suppress custom Limited → None, got {result:?}"
);
assert_eq!(
*calls.lock().unwrap(),
vec!["check_custom_limit"],
"underlying limiter must still be called in dry-run mode"
);
}
#[tokio::test]
async fn test_dry_run_allowed_stays_none() {
let (mock, _calls) = MockLimiter::new(
HashSet::new(),
EvalResult::Allowed,
EvalResult::NotApplicable,
);
let wrapper = GlobalRateLimiter::new_with_dry_run(mock);
let result = wrapper.is_limited("some_key", 1).await;
assert!(
result.is_none(),
"allowed result should remain None in dry-run mode"
);
}
#[tokio::test]
async fn test_non_dry_run_limited_still_returns_some() {
let (mock, _calls) = MockLimiter::new(
HashSet::new(),
make_limited_response(false),
EvalResult::NotApplicable,
);
let wrapper = GlobalRateLimiter::new_with(mock);
let result = wrapper.is_limited("some_key", 1).await;
assert!(
result.is_some(),
"non-dry-run should return Some(response) when limited"
);
}
// --- Hierarchical custom-key resolver ---
//
// The resolver is capture's key-resolution policy, injected into the common
// limiter. A lookup key is `token` or `token:distinct_id`. Resolution: exact
// match first, then the token prefix (everything before the first `:`). It is
// colon-delimited, NOT a string prefix match.
fn resolver_map(pairs: &[(&str, u64)]) -> HashMap<String, u64> {
pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
}
#[rstest]
// Token-scope override: applies to the token and ALL of its token:distinct_id keys.
#[case::token_scope_exact(&[("phc_tok", 10)], "phc_tok", Some(10))]
#[case::token_scope_matches_any_distinct_id(&[("phc_tok", 10)], "phc_tok:alice", Some(10))]
#[case::token_scope_matches_other_distinct_id(&[("phc_tok", 10)], "phc_tok:bob", Some(10))]
#[case::token_scope_other_token_unmatched(&[("phc_tok", 10)], "other:alice", None)]
#[case::token_scope_other_bare_token_unmatched(&[("phc_tok", 10)], "other", None)]
// token:distinct_id-scope override: applies to that exact pair only.
#[case::distinct_scope_exact(&[("phc_tok:alice", 5)], "phc_tok:alice", Some(5))]
#[case::distinct_scope_other_distinct_unmatched(&[("phc_tok:alice", 5)], "phc_tok:bob", None)]
#[case::distinct_scope_bare_token_unmatched(&[("phc_tok:alice", 5)], "phc_tok", None)]
// Exact match wins over the token-prefix fallback.
#[case::exact_wins_over_fallback(&[("phc_tok", 10), ("phc_tok:vip", 100)], "phc_tok:vip", Some(100))]
#[case::fallback_when_no_exact(&[("phc_tok", 10), ("phc_tok:vip", 100)], "phc_tok:other", Some(10))]
#[case::bare_token_with_both(&[("phc_tok", 10), ("phc_tok:vip", 100)], "phc_tok", Some(10))]
// Safety: fallback is colon-delimited, never a raw string prefix.
#[case::not_string_prefix_bare(&[("phc_tok", 10)], "phc_tokEXTRA", None)]
#[case::not_string_prefix_with_distinct(&[("phc_tok", 10)], "phc_tokEXTRA:alice", None)]
// Corner cases: empty distinct_id, extra colons, empty token, utf8.
#[case::empty_distinct_id_falls_back(&[("phc_tok", 10)], "phc_tok:", Some(10))]
#[case::only_first_colon_splits(&[("phc_tok", 10)], "phc_tok:a:b", Some(10))]
#[case::empty_token_key(&[("", 3)], ":alice", Some(3))]
#[case::utf8_distinct_id_falls_back(&[("phc_tok", 10)], "phc_tok:ünïcödé", Some(10))]
#[case::empty_map_no_match(&[], "phc_tok:alice", None)]
fn test_hierarchical_resolver(
#[case] pairs: &[(&str, u64)],
#[case] key: &str,
#[case] expected: Option<u64>,
) {
let resolver = GlobalRateLimiter::hierarchical_resolver();
let map = resolver_map(pairs);
assert_eq!(resolver(key, &map), expected, "key={key}");
}
#[tokio::test]
async fn test_static_csv_seed_resolves_hierarchically() {
// A static-CSV token-level override (dynamic source off) applies to that
// token's token:distinct_id keys too, because capture always sets the
// hierarchical resolver.
let limiter = GlobalRateLimiter::for_test_hierarchical_seeded(Some("phc_seed=7"));
assert!(limiter.is_custom_key("phc_seed"), "exact token override");
assert!(
limiter.is_custom_key("phc_seed:any_user"),
"token override must apply to token:distinct_id"
);
assert!(
!limiter.is_custom_key("other_tok:any_user"),
"unrelated token must not match"
);
}
}