:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use chrono::Utc;
use common_redis::{Client, CustomRedisError};
use dashmap::DashMap;
use metrics::gauge;
use std::sync::Arc;
use std::time::Duration;
use strum::Display;
use tokio::task;
use tokio::time::interval;
/// Limit resources by checking if a value is present in Redis
///
/// We have an async celery worker that regularly checks on accounts + assesses if they are beyond
/// a billing limit. If this is the case, a key is set in redis.
///
/// For replay sessions we also check if too many events are coming in in ingestion for a single session
/// and set a redis key to redirect further events to overflow.
///
/// For feature flag evaluations we still return a 200 response, but add an entry to the response body
/// to indicate that the flag evaluation was quota-limited.
///
/// Requirements
///
/// 1. Updates from the celery worker should be reflected in capture within a short period of time
/// 2. Quota limited services should cope with redis being _totally down_, and fail open
/// 3. We should not hit redis for every single request
///
/// The solution here is to read from the cache and update the set in a background thread.
/// We have to lock all readers briefly while we update the set, but we don't hold the lock
/// until we already have the response from redis so it should be very short.
///
/// Some small delay between an account being limited and the limit taking effect is acceptable.
/// However, ideally we should not allow requests from some pods but 429 from others.
// todo: fetch from env
// due to historical reasons we use different suffixes for quota limits and overflow
// hopefully we can unify these in the future
pub const QUOTA_LIMITER_CACHE_KEY: &str = "@posthog/quota-limits/";
pub const OVERFLOW_LIMITER_CACHE_KEY: &str = "@posthog/capture-overflow/";
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum QuotaResource {
Events,
Exceptions,
Recordings,
Replay,
FeatureFlags,
Surveys,
LLMEvents,
}
impl QuotaResource {
pub fn as_str(&self) -> &'static str {
match self {
Self::Events => "events",
Self::Exceptions => "exceptions",
Self::Recordings => "recordings",
Self::Replay => "replay",
Self::FeatureFlags => "feature_flag_requests",
Self::Surveys => "survey_responses",
Self::LLMEvents => "llm_events",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Display)]
pub enum ServiceName {
SessionReplay,
FeatureFlags,
Capture,
Cymbal,
}
impl ServiceName {
pub fn as_string(&self) -> String {
match self {
ServiceName::SessionReplay => "session_replay".to_string(),
ServiceName::FeatureFlags => "feature_flags".to_string(),
ServiceName::Capture => "capture".to_string(),
ServiceName::Cymbal => "cymbal".to_string(),
}
}
}
#[derive(Clone)]
pub struct RedisLimiter {
limited: Arc<DashMap<String, bool>>,
redis: Arc<dyn Client + Send + Sync>,
key: String,
interval: Duration,
service_name: ServiceName,
}
impl RedisLimiter {
/// Create a new RedisLimiter.
///
/// This connects to a redis cluster - pass in a vec of addresses for the initial nodes.
///
/// You can also initialize the limiter with a set of tokens to limit from the very beginning.
/// This may be overridden by Redis, if the sets differ,
///
/// Pass an empty redis node list to only use this initial set.
pub fn new(
interval: Duration,
redis: Arc<dyn Client + Send + Sync>,
limiter_cache_key: String,
redis_key_prefix: Option<String>,
resource: QuotaResource,
service_name: ServiceName,
) -> anyhow::Result<RedisLimiter> {
let limited = Arc::new(DashMap::new());
let key_prefix = redis_key_prefix.unwrap_or_default();
let limiter = RedisLimiter {
limited,
redis: redis.clone(),
key: format!("{key_prefix}{limiter_cache_key}{}", resource.as_str()),
interval,
service_name,
};
// Spawn a background task to periodically fetch data from Redis
limiter.spawn_background_update();
Ok(limiter)
}
fn spawn_background_update(&self) {
let limited = Arc::clone(&self.limited);
let redis = Arc::clone(&self.redis);
let interval_duration = self.interval;
let key = self.key.clone();
let service_name = self.service_name.as_string();
// Spawn a task to periodically update the cache from Redis
task::spawn(async move {
let mut interval = interval(interval_duration);
loop {
match RedisLimiter::fetch_limited(&redis, &key).await {
Ok(set) => {
gauge!(
format!("{}_billing_limits_loaded_tokens", service_name),
"cache_key" => key.clone(),
)
.set(set.len() as f64);
limited.clear();
for item in set {
limited.insert(item, true);
}
}
Err(e) => {
tracing::warn!("Failed to update cache from Redis: {:?}", e);
}
}
interval.tick().await;
}
});
}
async fn fetch_limited(
client: &Arc<dyn Client + Send + Sync>,
key: &String,
) -> anyhow::Result<Vec<String>, CustomRedisError> {
let now = Utc::now().timestamp();
client
.zrangebyscore(key.to_string(), now.to_string(), String::from("+Inf"))
.await
}
pub async fn is_limited(&self, value: &str) -> bool {
self.limited.get(value).is_some()
}
}
#[cfg(test)]
mod tests {
use super::{OVERFLOW_LIMITER_CACHE_KEY, QUOTA_LIMITER_CACHE_KEY};
use crate::redis::{QuotaResource, RedisLimiter, ServiceName};
use common_redis::MockRedisClient;
use std::{sync::Arc, time::Duration};
#[tokio::test]
async fn test_dynamic_limited() {
let client = MockRedisClient::new().zrangebyscore_ret(
"@posthog/capture-overflow/replay",
vec![String::from("banana")],
);
let client = Arc::new(client);
let limiter = RedisLimiter::new(
Duration::from_secs(1),
client,
OVERFLOW_LIMITER_CACHE_KEY.to_string(),
None,
QuotaResource::Replay,
ServiceName::Capture,
)
.expect("Failed to create billing limiter");
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
assert!(!limiter.is_limited("not_limited").await);
assert!(limiter.is_limited("banana").await);
}
#[tokio::test]
async fn test_custom_key_prefix() {
let client = MockRedisClient::new().zrangebyscore_ret(
"prefix//@posthog/quota-limits/events",
vec![String::from("banana")],
);
let client = Arc::new(client);
// Default lookup without prefix fails
let limiter = RedisLimiter::new(
Duration::from_secs(1),
client.clone(),
QUOTA_LIMITER_CACHE_KEY.to_string(),
None,
QuotaResource::Events,
ServiceName::Capture,
)
.expect("Failed to create billing limiter");
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
assert!(!limiter.is_limited("banana").await);
// Limiter using the correct prefix
let prefixed_limiter = RedisLimiter::new(
Duration::from_micros(1),
client,
QUOTA_LIMITER_CACHE_KEY.to_string(),
Some("prefix//".to_string()),
QuotaResource::Events,
ServiceName::Capture,
)
.expect("Failed to create billing limiter");
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
assert!(!prefixed_limiter.is_limited("not_limited").await);
assert!(prefixed_limiter.is_limited("banana").await);
}
#[tokio::test]
async fn test_feature_flag_limiter() {
let client = MockRedisClient::new().zrangebyscore_ret(
"@posthog/quota-limits/feature_flag_requests",
vec![String::from("banana")],
);
let client = Arc::new(client);
let limiter = RedisLimiter::new(
Duration::from_secs(1),
client,
QUOTA_LIMITER_CACHE_KEY.to_string(),
None,
QuotaResource::FeatureFlags,
ServiceName::FeatureFlags,
)
.expect("Failed to create feature flag limiter");
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
assert!(!limiter.is_limited("not_limited").await);
assert!(limiter.is_limited("banana").await);
}
}