:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use common_redis::CustomRedisError;
use futures::future::join_all;
use metrics::gauge;
use tokio::sync::RwLock;
use tokio::time::interval;
use tracing::{error, info, warn};
use super::repository::EventRestrictionsRepository;
use super::types::{
AppliedRestrictions, EventContext, Pipeline, Restriction, RestrictionSet, RestrictionType,
};
/// Manages restrictions by pipeline and token.
///
/// A single `RestrictionManager` can serve multiple pipelines (e.g. the events
/// capture deployment serves both `Analytics` and `ErrorTracking`). An entry
/// that applies to multiple pipelines is indexed under each — lookup is
/// pipeline-then-token, not token-then-pipeline.
#[derive(Debug, Clone, Default)]
pub struct RestrictionManager {
pub restrictions: HashMap<Pipeline, HashMap<String, Vec<Restriction>>>,
}
impl RestrictionManager {
pub fn new() -> Self {
Self::default()
}
/// Get all restriction types that apply to an event for a given pipeline.
pub fn get_restrictions(
&self,
token: &str,
event: &EventContext,
pipeline: Pipeline,
) -> RestrictionSet {
let Some(restrictions) = self
.restrictions
.get(&pipeline)
.and_then(|by_token| by_token.get(token))
else {
return RestrictionSet::new();
};
let mut result = RestrictionSet::new();
for r in restrictions {
if r.matches(event) {
match r.restriction_type {
RestrictionType::RedirectToTopic => {
if let Some(topic) = r
.args
.as_ref()
.and_then(|a| a.get("topic"))
.and_then(|t| t.as_str())
{
if !topic.is_empty() {
result.insert_redirect_to_topic(topic.to_string());
}
}
}
other => result.insert(other),
}
}
}
result
}
/// Test/setup helper: insert restrictions for a single (pipeline, token) pair.
pub fn insert_restrictions(
&mut self,
pipeline: Pipeline,
token: impl Into<String>,
restrictions: Vec<Restriction>,
) {
self.restrictions
.entry(pipeline)
.or_default()
.insert(token.into(), restrictions);
}
/// Build a RestrictionManager from repository data for the given allowlist
/// of pipelines. Entries that don't apply to any allowed pipeline are
/// dropped at fetch time; entries applying to multiple allowed pipelines
/// are indexed under each.
///
/// Returns an error if any restriction type fails to fetch, which signals
/// a likely dead Redis connection and triggers a reconnect.
pub async fn from_repository(
repository: &dyn EventRestrictionsRepository,
allowed_pipelines: &[Pipeline],
) -> Result<Self, CustomRedisError> {
info!(
pipelines = ?allowed_pipelines,
"Fetching event restrictions"
);
let mut manager = Self::new();
// Fetch all restriction types in parallel
let fetch_futures = RestrictionType::all()
.into_iter()
.map(|rt| async move { (rt, repository.get_entries(rt).await) });
let results = join_all(fetch_futures).await;
let mut fetch_error: Option<CustomRedisError> = None;
for (restriction_type, result) in results {
let mut entries = match result {
Ok(Some(e)) => e,
Ok(None) => continue,
Err(e) => {
warn!(
restriction_type = %restriction_type.as_str(),
error = %e,
"Failed to fetch restriction entries"
);
fetch_error = Some(e);
continue;
}
};
// Sort by index ascending for deterministic ordering
entries.sort_by_key(|e| e.index.unwrap_or(0));
for entry in entries {
// Skip old format entries (version must be 2)
if entry.version != Some(2) {
continue;
}
// Resolve which allowed pipelines this entry applies to.
let entry_pipelines: Vec<Pipeline> = entry
.pipelines
.iter()
.filter_map(|s| Pipeline::parse(s))
.filter(|p| allowed_pipelines.contains(p))
.collect();
if entry_pipelines.is_empty() {
continue;
}
// For redirect_to_topic, validate args.topic exists
if restriction_type == RestrictionType::RedirectToTopic {
let has_valid_topic = entry
.args
.as_ref()
.and_then(|a| a.get("topic"))
.and_then(|t| t.as_str())
.map(|t| !t.is_empty())
.unwrap_or(false);
if !has_valid_topic {
error!(
token = %entry.token,
"redirect_to_topic restriction missing valid args.topic, skipping"
);
continue;
}
}
let token = entry.token.clone();
let restriction = entry.into_restriction(restriction_type);
for pipeline in entry_pipelines {
manager
.restrictions
.entry(pipeline)
.or_default()
.entry(token.clone())
.or_default()
.push(restriction.clone());
}
}
}
if let Some(e) = fetch_error {
return Err(e);
}
for pipeline in allowed_pipelines {
let by_token = manager.restrictions.get(pipeline);
let restriction_count: usize = by_token
.map(|m| m.values().map(|v| v.len()).sum())
.unwrap_or(0);
let token_count: usize = by_token.map(|m| m.len()).unwrap_or(0);
info!(
pipeline = %pipeline.as_str(),
total_restrictions = restriction_count,
total_tokens = token_count,
"Fetched event restrictions"
);
}
Ok(manager)
}
}
/// Service that manages event restrictions with background refresh and fail-open behavior.
///
/// One service serves all pipelines its host capture deployment produces to —
/// the events deployment uses `[Analytics, ErrorTracking]`, replay uses
/// `[SessionRecordings]`, ai uses `[Ai]`. Callers select the right pipeline
/// per event when calling [`Self::get_restrictions`].
#[derive(Clone)]
pub struct EventRestrictionService {
manager: Arc<RwLock<RestrictionManager>>,
last_successful_refresh: Arc<AtomicI64>,
fail_open_after: Duration,
pipelines: Vec<Pipeline>,
}
impl EventRestrictionService {
/// Create a new service. Call `start_refresh_task` to begin background updates.
pub fn new(pipelines: Vec<Pipeline>, fail_open_after: Duration) -> Self {
Self {
manager: Arc::new(RwLock::new(RestrictionManager::new())),
last_successful_refresh: Arc::new(AtomicI64::new(0)),
fail_open_after,
pipelines,
}
}
/// Refreshes restrictions periodically until the cancellation token is triggered.
///
/// The repository is created lazily via `create_repository`. If Redis is unavailable
/// at startup, the service stays in fail-open mode and retries on each tick.
/// `tokio::time::interval` ticks immediately, so the first connection attempt
/// happens without delay.
pub async fn start_refresh_task<F, Fut>(
&self,
create_repository: F,
refresh_interval: Duration,
shutdown_handle: lifecycle::Handle,
) where
F: Fn() -> Fut,
Fut: std::future::Future<
Output = Result<Arc<dyn EventRestrictionsRepository>, common_redis::CustomRedisError>,
>,
{
let mut interval = interval(refresh_interval);
let mut repository: Option<Arc<dyn EventRestrictionsRepository>> = None;
loop {
tokio::select! {
_ = shutdown_handle.shutdown_recv() => {
info!(pipelines = ?self.pipelines, "Event restrictions refresh task shutting down");
break;
}
_ = interval.tick() => {
if repository.is_none() {
match create_repository().await {
Ok(repo) => {
info!(pipelines = ?self.pipelines, "Event restrictions connected to Redis");
repository = Some(repo);
}
Err(e) => {
error!(
pipelines = ?self.pipelines,
error = %e,
"Failed to connect to event restrictions Redis, will retry"
);
continue;
}
}
}
if !self.refresh_from_repository(repository.as_ref().unwrap().as_ref()).await {
// All fetches failed — connection is likely dead, will reconnect next tick
repository = None;
}
}
}
}
}
/// Fetch restrictions from repository and update the local cache.
/// Returns `true` on success, `false` if all fetches failed (dead connection).
async fn refresh_from_repository(&self, repository: &dyn EventRestrictionsRepository) -> bool {
match RestrictionManager::from_repository(repository, &self.pipelines).await {
Ok(new_manager) => {
// Count per-pipeline before handing the manager off to update().
let counts: Vec<(Pipeline, usize, usize)> = self
.pipelines
.iter()
.map(|pipeline| {
let by_token = new_manager.restrictions.get(pipeline);
let restriction_count: usize = by_token
.map(|m| m.values().map(|v| v.len()).sum())
.unwrap_or(0);
let token_count: usize = by_token.map(|m| m.len()).unwrap_or(0);
(*pipeline, restriction_count, token_count)
})
.collect();
let now = self.update(new_manager).await;
for (pipeline, restriction_count, token_count) in counts {
let pipeline_str = pipeline.as_str();
gauge!(
"capture_event_restrictions_last_refresh_timestamp",
"pipeline" => pipeline_str.to_string()
)
.set(now as f64);
gauge!(
"capture_event_restrictions_loaded_count",
"pipeline" => pipeline_str.to_string()
)
.set(restriction_count as f64);
gauge!(
"capture_event_restrictions_tokens_count",
"pipeline" => pipeline_str.to_string()
)
.set(token_count as f64);
gauge!(
"capture_event_restrictions_stale",
"pipeline" => pipeline_str.to_string()
)
.set(0.0);
}
true
}
Err(e) => {
error!(
pipelines = ?self.pipelines,
error = %e,
"Failed to refresh event restrictions, will reconnect"
);
false
}
}
}
/// Manually update restrictions (useful for testing). Returns the timestamp.
pub async fn update(&self, new_manager: RestrictionManager) -> i64 {
let mut guard = self.manager.write().await;
*guard = new_manager;
let now = chrono::Utc::now().timestamp();
self.last_successful_refresh.store(now, Ordering::SeqCst);
now
}
/// Whether at least one refresh has succeeded since boot. Until then the
/// service is fail-open and every lookup returns empty, so callers that
/// need loaded state (tests synchronizing with the initial load) poll this
/// instead of guessing at refresh timing.
pub fn has_loaded(&self) -> bool {
self.last_successful_refresh.load(Ordering::SeqCst) != 0
}
/// Check if the cache is stale (fail-open should be active).
fn is_stale_at(&self, now_ts: i64) -> bool {
let last_refresh = self.last_successful_refresh.load(Ordering::SeqCst);
if last_refresh == 0 {
return true;
}
// Use saturating_sub to handle potential clock skew (if now < last_refresh, age = 0)
let age_secs = (now_ts as u64).saturating_sub(last_refresh as u64);
Duration::from_secs(age_secs) > self.fail_open_after
}
/// Get applied restrictions for an event under the given pipeline. Returns
/// empty if fail-open is active or if the pipeline isn't in the service's
/// allowlist (no entries are indexed for it).
pub async fn get_restrictions(
&self,
token: &str,
event: &EventContext<'_>,
pipeline: Pipeline,
) -> AppliedRestrictions {
if self.is_stale_at(event.now_ts) {
gauge!(
"capture_event_restrictions_stale",
"pipeline" => pipeline.as_str().to_string()
)
.set(1.0);
return AppliedRestrictions::default();
}
let guard = self.manager.read().await;
let set = guard.get_restrictions(token, event, pipeline);
AppliedRestrictions::from_restrictions(set, pipeline)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::CaptureMode;
use crate::event_restrictions::repository::testing::MockRestrictionsRepository;
use crate::event_restrictions::repository::RestrictionEntry;
use crate::event_restrictions::types::RestrictionScope;
use crate::v0_request::DataType;
use common_redis::CustomRedisError;
use strum::IntoEnumIterator;
use tokio_util::sync::CancellationToken;
fn test_lifecycle() -> (CancellationToken, lifecycle::Handle) {
let shutdown_token = CancellationToken::new();
let mut manager = lifecycle::Manager::builder("test")
.with_trap_signals(false)
.with_prestop_check(false)
.with_shutdown_token(shutdown_token.clone())
.build();
let handle = manager.register("event-restrictions", lifecycle::ComponentOptions::new());
let _monitor = manager.monitor_background();
(shutdown_token, handle)
}
fn make_entry(token: &str, pipelines: Vec<&str>) -> RestrictionEntry {
RestrictionEntry {
version: Some(2),
token: token.to_string(),
pipelines: pipelines.into_iter().map(|s| s.to_string()).collect(),
distinct_ids: vec![],
session_ids: vec![],
event_names: vec![],
event_uuids: vec![],
args: None,
index: None,
}
}
fn make_entry_with_filters(
token: &str,
pipelines: Vec<&str>,
distinct_ids: Vec<&str>,
event_names: Vec<&str>,
) -> RestrictionEntry {
RestrictionEntry {
version: Some(2),
token: token.to_string(),
pipelines: pipelines.into_iter().map(|s| s.to_string()).collect(),
distinct_ids: distinct_ids.into_iter().map(|s| s.to_string()).collect(),
session_ids: vec![],
event_names: event_names.into_iter().map(|s| s.to_string()).collect(),
event_uuids: vec![],
args: None,
index: None,
}
}
#[tokio::test]
async fn test_manager_applies_all_restriction_types() {
let repo = MockRestrictionsRepository::new();
// Set up one entry for each restriction type
repo.set_entries(
RestrictionType::DropEvent,
Some(vec![make_entry("token_drop", vec!["analytics"])]),
)
.await;
repo.set_entries(
RestrictionType::ForceOverflow,
Some(vec![make_entry("token_overflow", vec!["analytics"])]),
)
.await;
repo.set_entries(
RestrictionType::RedirectToDlq,
Some(vec![make_entry("token_dlq", vec!["analytics"])]),
)
.await;
repo.set_entries(
RestrictionType::SkipPersonProcessing,
Some(vec![make_entry("token_skip", vec!["analytics"])]),
)
.await;
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
let event = EventContext::default();
// Each token should have exactly its restriction type
let drop_restrictions = manager.get_restrictions("token_drop", &event, Pipeline::Analytics);
assert_eq!(drop_restrictions.len(), 1);
assert!(drop_restrictions.contains(RestrictionType::DropEvent));
let overflow_restrictions =
manager.get_restrictions("token_overflow", &event, Pipeline::Analytics);
assert_eq!(overflow_restrictions.len(), 1);
assert!(overflow_restrictions.contains(RestrictionType::ForceOverflow));
let dlq_restrictions = manager.get_restrictions("token_dlq", &event, Pipeline::Analytics);
assert_eq!(dlq_restrictions.len(), 1);
assert!(dlq_restrictions.contains(RestrictionType::RedirectToDlq));
let skip_restrictions = manager.get_restrictions("token_skip", &event, Pipeline::Analytics);
assert_eq!(skip_restrictions.len(), 1);
assert!(skip_restrictions.contains(RestrictionType::SkipPersonProcessing));
// Unknown token should have no restrictions
let unknown = manager.get_restrictions("unknown_token", &event, Pipeline::Analytics);
assert!(unknown.is_empty());
}
#[tokio::test]
async fn test_manager_applies_filters_correctly() {
let repo = MockRestrictionsRepository::new();
repo.set_entries(
RestrictionType::DropEvent,
Some(vec![make_entry_with_filters(
"token1",
vec!["analytics"],
vec!["user1", "user2"],
vec!["$pageview"],
)]),
)
.await;
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
// Should match when both distinct_id and event_name match
let event_match = EventContext {
distinct_id: Some("user1"),
event_name: Some("$pageview"),
..Default::default()
};
let restrictions = manager.get_restrictions("token1", &event_match, Pipeline::Analytics);
assert!(restrictions.contains(RestrictionType::DropEvent));
// Should NOT match when event_name doesn't match (AND logic)
let event_wrong_name = EventContext {
distinct_id: Some("user1"),
event_name: Some("$identify"),
..Default::default()
};
let restrictions =
manager.get_restrictions("token1", &event_wrong_name, Pipeline::Analytics);
assert!(restrictions.is_empty());
// Should NOT match when distinct_id doesn't match
let event_wrong_user = EventContext {
distinct_id: Some("user3"),
event_name: Some("$pageview"),
..Default::default()
};
let restrictions =
manager.get_restrictions("token1", &event_wrong_user, Pipeline::Analytics);
assert!(restrictions.is_empty());
}
#[tokio::test]
async fn test_manager_filters_by_pipeline() {
let repo = MockRestrictionsRepository::new();
repo.set_entries(
RestrictionType::DropEvent,
Some(vec![
make_entry("token_analytics", vec!["analytics"]),
make_entry("token_recordings", vec!["session_recordings"]),
make_entry("token_both", vec!["analytics", "session_recordings"]),
]),
)
.await;
// Fetch for analytics pipeline
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
let event = EventContext::default();
// analytics token should be present
assert!(!manager
.get_restrictions("token_analytics", &event, Pipeline::Analytics)
.is_empty());
// recordings token should NOT be present (entry not indexed under
// Analytics, even though it exists under SessionRecordings — and
// SessionRecordings isn't in the allowlist).
assert!(manager
.get_restrictions("token_recordings", &event, Pipeline::Analytics)
.is_empty());
// both token should be present
assert!(!manager
.get_restrictions("token_both", &event, Pipeline::Analytics)
.is_empty());
}
#[tokio::test]
async fn events_deployment_loads_restrictions_for_every_classified_pipeline() {
for data_type in DataType::iter() {
let Some(pipeline) = data_type.pipeline() else {
continue;
};
let repo = MockRestrictionsRepository::new();
repo.set_entries(
RestrictionType::DropEvent,
Some(vec![make_entry("token", vec![pipeline.as_str()])]),
)
.await;
let manager = RestrictionManager::from_repository(
&repo,
&Pipeline::for_capture_mode(CaptureMode::Events),
)
.await
.unwrap();
let restrictions =
manager.get_restrictions("token", &EventContext::default(), pipeline);
assert!(
restrictions.contains(RestrictionType::DropEvent),
"{data_type:?}"
);
}
}
#[tokio::test]
async fn test_manager_skips_old_version() {
let repo = MockRestrictionsRepository::new();
let mut old_entry = make_entry("token_old", vec!["analytics"]);
old_entry.version = Some(1); // Old version
let new_entry = make_entry("token_new", vec!["analytics"]);
repo.set_entries(RestrictionType::DropEvent, Some(vec![old_entry, new_entry]))
.await;
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
let event = EventContext::default();
// Old version should be skipped
assert!(manager
.get_restrictions("token_old", &event, Pipeline::Analytics)
.is_empty());
// New version should be present
assert!(!manager
.get_restrictions("token_new", &event, Pipeline::Analytics)
.is_empty());
}
#[tokio::test]
async fn test_manager_returns_error_when_any_fetch_fails() {
let repo = MockRestrictionsRepository::new();
repo.set_error(RestrictionType::DropEvent, CustomRedisError::Timeout)
.await;
repo.set_entries(
RestrictionType::ForceOverflow,
Some(vec![make_entry("token1", vec!["analytics"])]),
)
.await;
let result = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics]).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_manager_handles_empty_repository() {
let repo = MockRestrictionsRepository::new();
// Don't set any entries
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
assert!(manager.restrictions.is_empty());
}
#[tokio::test]
async fn test_manager_multiple_restrictions_same_token() {
let repo = MockRestrictionsRepository::new();
// Same token has multiple restriction types
repo.set_entries(
RestrictionType::ForceOverflow,
Some(vec![make_entry("token1", vec!["analytics"])]),
)
.await;
repo.set_entries(
RestrictionType::SkipPersonProcessing,
Some(vec![make_entry("token1", vec!["analytics"])]),
)
.await;
let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
.await
.unwrap();
let restrictions =
manager.get_restrictions("token1", &EventContext::default(), Pipeline::Analytics);
assert_eq!(restrictions.len(), 2);
assert!(restrictions.contains(RestrictionType::ForceOverflow));
assert!(restrictions.contains(RestrictionType::SkipPersonProcessing));
}
// ========================================================================
// EventRestrictionService tests
// ========================================================================
fn event_ctx_now() -> EventContext<'static> {
EventContext {
now_ts: chrono::Utc::now().timestamp(),
..Default::default()
}
}
#[tokio::test]
async fn test_service_is_stale_when_never_refreshed() {
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
// last_successful_refresh is 0, so should be stale (fail-open)
let applied = service
.get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.is_empty());
}
#[tokio::test]
async fn test_service_returns_restrictions_after_update() {
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
let mut manager = RestrictionManager::new();
manager.insert_restrictions(
Pipeline::Analytics,
"token1",
vec![Restriction {
restriction_type: RestrictionType::DropEvent,
scope: RestrictionScope::AllEvents,
args: None,
}],
);
service.update(manager).await;
let applied = service
.get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.should_drop());
}
#[tokio::test]
async fn test_service_fail_open_after_timeout() {
let service = EventRestrictionService::new(
vec![Pipeline::Analytics],
Duration::from_secs(1), // 1 second timeout
);
let mut manager = RestrictionManager::new();
manager.insert_restrictions(
Pipeline::Analytics,
"token1",
vec![Restriction {
restriction_type: RestrictionType::DropEvent,
scope: RestrictionScope::AllEvents,
args: None,
}],
);
service.update(manager).await;
// Immediately after update, should return restrictions
let applied = service
.get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.should_drop());
// Manually set last_successful_refresh to 10 seconds ago
let old_timestamp = chrono::Utc::now().timestamp() - 10;
service
.last_successful_refresh
.store(old_timestamp, Ordering::SeqCst);
// Now should be stale (fail-open)
let applied_after = service
.get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied_after.is_empty());
}
// ========================================================================
// start_refresh_task tests
// ========================================================================
#[tokio::test]
async fn test_refresh_task_loads_restrictions() {
let (shutdown_token, lifecycle_handle) = test_lifecycle();
let shutdown_token_clone = shutdown_token.clone();
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
let service_clone = service.clone();
let handle = tokio::spawn(async move {
service_clone
.start_refresh_task(
move || {
let st = shutdown_token_clone.clone();
async move {
st.cancel();
let repo = MockRestrictionsRepository::new();
repo.set_entries(
RestrictionType::DropEvent,
Some(vec![make_entry("token1", vec!["analytics"])]),
)
.await;
let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
Ok(result)
}
},
Duration::from_millis(5),
lifecycle_handle,
)
.await;
});
handle.await.unwrap();
let applied = service
.get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.should_drop());
let unknown = service
.get_restrictions("unknown", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(unknown.is_empty());
}
#[tokio::test]
async fn test_refresh_task_retries_connection_while_fail_open() {
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
let (shutdown_token, lifecycle_handle) = test_lifecycle();
let shutdown_token_clone = shutdown_token.clone();
let attempt_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
let count_clone = attempt_count.clone();
let service_clone = service.clone();
let handle = tokio::spawn(async move {
service_clone
.start_refresh_task(
move || {
let n = count_clone.fetch_add(1, Ordering::SeqCst);
let st = shutdown_token_clone.clone();
async move {
if n >= 2 {
st.cancel();
}
Err(CustomRedisError::Timeout)
}
},
Duration::from_millis(5),
lifecycle_handle,
)
.await;
});
handle.await.unwrap();
let applied = service
.get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.is_empty());
assert!(
attempt_count.load(Ordering::SeqCst) >= 2,
"should have retried"
);
}
#[tokio::test]
async fn test_refresh_task_reconnects_after_refresh_failure() {
let (shutdown_token, lifecycle_handle) = test_lifecycle();
let shutdown_token_clone = shutdown_token.clone();
let connect_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
let count_clone = connect_count.clone();
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
let service_clone = service.clone();
let handle = tokio::spawn(async move {
service_clone
.start_refresh_task(
move || {
let n = count_clone.fetch_add(1, Ordering::SeqCst);
let st = shutdown_token_clone.clone();
async move {
// First connection succeeds but repo returns all errors,
// second connection triggers shutdown so we can assert.
if n >= 1 {
st.cancel();
}
let repo = MockRestrictionsRepository::new();
for rt in RestrictionType::all() {
repo.set_error(rt, CustomRedisError::Timeout).await;
}
let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
Ok(result)
}
},
Duration::from_millis(5),
lifecycle_handle,
)
.await;
});
handle.await.unwrap();
assert!(
connect_count.load(Ordering::SeqCst) >= 2,
"should have reconnected after refresh failure"
);
// Service never got a successful refresh, so it stays fail-open
let applied = service
.get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(applied.is_empty());
}
#[tokio::test]
async fn test_refresh_task_fail_open_then_recover() {
let (shutdown_token, lifecycle_handle) = test_lifecycle();
let shutdown_token_clone = shutdown_token.clone();
let attempt_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
let count_clone = attempt_count.clone();
let service =
EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
let service_clone = service.clone();
let handle = tokio::spawn(async move {
service_clone
.start_refresh_task(
move || {
let n = count_clone.fetch_add(1, Ordering::SeqCst);
let st = shutdown_token_clone.clone();
async move {
if n < 2 {
// First two attempts fail (simulating Redis down)
return Err(CustomRedisError::Timeout);
}
// Third attempt succeeds with restrictions
st.cancel();
let repo = MockRestrictionsRepository::new();
repo.set_entries(
RestrictionType::ForceOverflow,
Some(vec![make_entry("token1", vec!["analytics"])]),
)
.await;
let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
Ok(result)
}
},
Duration::from_millis(5),
lifecycle_handle,
)
.await;
});
handle.await.unwrap();
assert!(
attempt_count.load(Ordering::SeqCst) >= 3,
"should have attempted at least 3 times"
);
// After recovery, restrictions should be active
let applied = service
.get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
.await;
assert!(
applied.force_overflow(),
"restrictions should be active after recovery"
);
}
}