Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

35.8 KB
Raw Blame
use std::collections::HashMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::Arc;
use std::time::Duration;

use common_redis::CustomRedisError;
use futures::future::join_all;
use metrics::gauge;
use tokio::sync::RwLock;
use tokio::time::interval;
use tracing::{error, info, warn};

use super::repository::EventRestrictionsRepository;
use super::types::{
    AppliedRestrictions, EventContext, Pipeline, Restriction, RestrictionSet, RestrictionType,
};

/// Manages restrictions by pipeline and token.
///
/// A single `RestrictionManager` can serve multiple pipelines (e.g. the events
/// capture deployment serves both `Analytics` and `ErrorTracking`). An entry
/// that applies to multiple pipelines is indexed under each — lookup is
/// pipeline-then-token, not token-then-pipeline.
#[derive(Debug, Clone, Default)]
pub struct RestrictionManager {
    pub restrictions: HashMap<Pipeline, HashMap<String, Vec<Restriction>>>,
}

impl RestrictionManager {
    pub fn new() -> Self {
        Self::default()
    }

    /// Get all restriction types that apply to an event for a given pipeline.
    pub fn get_restrictions(
        &self,
        token: &str,
        event: &EventContext,
        pipeline: Pipeline,
    ) -> RestrictionSet {
        let Some(restrictions) = self
            .restrictions
            .get(&pipeline)
            .and_then(|by_token| by_token.get(token))
        else {
            return RestrictionSet::new();
        };

        let mut result = RestrictionSet::new();
        for r in restrictions {
            if r.matches(event) {
                match r.restriction_type {
                    RestrictionType::RedirectToTopic => {
                        if let Some(topic) = r
                            .args
                            .as_ref()
                            .and_then(|a| a.get("topic"))
                            .and_then(|t| t.as_str())
                        {
                            if !topic.is_empty() {
                                result.insert_redirect_to_topic(topic.to_string());
                            }
                        }
                    }
                    other => result.insert(other),
                }
            }
        }
        result
    }

    /// Test/setup helper: insert restrictions for a single (pipeline, token) pair.
    pub fn insert_restrictions(
        &mut self,
        pipeline: Pipeline,
        token: impl Into<String>,
        restrictions: Vec<Restriction>,
    ) {
        self.restrictions
            .entry(pipeline)
            .or_default()
            .insert(token.into(), restrictions);
    }

    /// Build a RestrictionManager from repository data for the given allowlist
    /// of pipelines. Entries that don't apply to any allowed pipeline are
    /// dropped at fetch time; entries applying to multiple allowed pipelines
    /// are indexed under each.
    ///
    /// Returns an error if any restriction type fails to fetch, which signals
    /// a likely dead Redis connection and triggers a reconnect.
    pub async fn from_repository(
        repository: &dyn EventRestrictionsRepository,
        allowed_pipelines: &[Pipeline],
    ) -> Result<Self, CustomRedisError> {
        info!(
            pipelines = ?allowed_pipelines,
            "Fetching event restrictions"
        );

        let mut manager = Self::new();

        // Fetch all restriction types in parallel
        let fetch_futures = RestrictionType::all()
            .into_iter()
            .map(|rt| async move { (rt, repository.get_entries(rt).await) });

        let results = join_all(fetch_futures).await;

        let mut fetch_error: Option<CustomRedisError> = None;

        for (restriction_type, result) in results {
            let mut entries = match result {
                Ok(Some(e)) => e,
                Ok(None) => continue,
                Err(e) => {
                    warn!(
                        restriction_type = %restriction_type.as_str(),
                        error = %e,
                        "Failed to fetch restriction entries"
                    );
                    fetch_error = Some(e);
                    continue;
                }
            };

            // Sort by index ascending for deterministic ordering
            entries.sort_by_key(|e| e.index.unwrap_or(0));

            for entry in entries {
                // Skip old format entries (version must be 2)
                if entry.version != Some(2) {
                    continue;
                }

                // Resolve which allowed pipelines this entry applies to.
                let entry_pipelines: Vec<Pipeline> = entry
                    .pipelines
                    .iter()
                    .filter_map(|s| Pipeline::parse(s))
                    .filter(|p| allowed_pipelines.contains(p))
                    .collect();
                if entry_pipelines.is_empty() {
                    continue;
                }

                // For redirect_to_topic, validate args.topic exists
                if restriction_type == RestrictionType::RedirectToTopic {
                    let has_valid_topic = entry
                        .args
                        .as_ref()
                        .and_then(|a| a.get("topic"))
                        .and_then(|t| t.as_str())
                        .map(|t| !t.is_empty())
                        .unwrap_or(false);
                    if !has_valid_topic {
                        error!(
                            token = %entry.token,
                            "redirect_to_topic restriction missing valid args.topic, skipping"
                        );
                        continue;
                    }
                }

                let token = entry.token.clone();
                let restriction = entry.into_restriction(restriction_type);
                for pipeline in entry_pipelines {
                    manager
                        .restrictions
                        .entry(pipeline)
                        .or_default()
                        .entry(token.clone())
                        .or_default()
                        .push(restriction.clone());
                }
            }
        }

        if let Some(e) = fetch_error {
            return Err(e);
        }

        for pipeline in allowed_pipelines {
            let by_token = manager.restrictions.get(pipeline);
            let restriction_count: usize = by_token
                .map(|m| m.values().map(|v| v.len()).sum())
                .unwrap_or(0);
            let token_count: usize = by_token.map(|m| m.len()).unwrap_or(0);
            info!(
                pipeline = %pipeline.as_str(),
                total_restrictions = restriction_count,
                total_tokens = token_count,
                "Fetched event restrictions"
            );
        }

        Ok(manager)
    }
}

/// Service that manages event restrictions with background refresh and fail-open behavior.
///
/// One service serves all pipelines its host capture deployment produces to —
/// the events deployment uses `[Analytics, ErrorTracking]`, replay uses
/// `[SessionRecordings]`, ai uses `[Ai]`. Callers select the right pipeline
/// per event when calling [`Self::get_restrictions`].
#[derive(Clone)]
pub struct EventRestrictionService {
    manager: Arc<RwLock<RestrictionManager>>,
    last_successful_refresh: Arc<AtomicI64>,
    fail_open_after: Duration,
    pipelines: Vec<Pipeline>,
}

impl EventRestrictionService {
    /// Create a new service. Call `start_refresh_task` to begin background updates.
    pub fn new(pipelines: Vec<Pipeline>, fail_open_after: Duration) -> Self {
        Self {
            manager: Arc::new(RwLock::new(RestrictionManager::new())),
            last_successful_refresh: Arc::new(AtomicI64::new(0)),
            fail_open_after,
            pipelines,
        }
    }

    /// Refreshes restrictions periodically until the cancellation token is triggered.
    ///
    /// The repository is created lazily via `create_repository`. If Redis is unavailable
    /// at startup, the service stays in fail-open mode and retries on each tick.
    /// `tokio::time::interval` ticks immediately, so the first connection attempt
    /// happens without delay.
    pub async fn start_refresh_task<F, Fut>(
        &self,
        create_repository: F,
        refresh_interval: Duration,
        shutdown_handle: lifecycle::Handle,
    ) where
        F: Fn() -> Fut,
        Fut: std::future::Future<
            Output = Result<Arc<dyn EventRestrictionsRepository>, common_redis::CustomRedisError>,
        >,
    {
        let mut interval = interval(refresh_interval);
        let mut repository: Option<Arc<dyn EventRestrictionsRepository>> = None;

        loop {
            tokio::select! {
                _ = shutdown_handle.shutdown_recv() => {
                    info!(pipelines = ?self.pipelines, "Event restrictions refresh task shutting down");
                    break;
                }
                _ = interval.tick() => {
                    if repository.is_none() {
                        match create_repository().await {
                            Ok(repo) => {
                                info!(pipelines = ?self.pipelines, "Event restrictions connected to Redis");
                                repository = Some(repo);
                            }
                            Err(e) => {
                                error!(
                                    pipelines = ?self.pipelines,
                                    error = %e,
                                    "Failed to connect to event restrictions Redis, will retry"
                                );
                                continue;
                            }
                        }
                    }

                    if !self.refresh_from_repository(repository.as_ref().unwrap().as_ref()).await {
                        // All fetches failed — connection is likely dead, will reconnect next tick
                        repository = None;
                    }
                }
            }
        }
    }

    /// Fetch restrictions from repository and update the local cache.
    /// Returns `true` on success, `false` if all fetches failed (dead connection).
    async fn refresh_from_repository(&self, repository: &dyn EventRestrictionsRepository) -> bool {
        match RestrictionManager::from_repository(repository, &self.pipelines).await {
            Ok(new_manager) => {
                // Count per-pipeline before handing the manager off to update().
                let counts: Vec<(Pipeline, usize, usize)> = self
                    .pipelines
                    .iter()
                    .map(|pipeline| {
                        let by_token = new_manager.restrictions.get(pipeline);
                        let restriction_count: usize = by_token
                            .map(|m| m.values().map(|v| v.len()).sum())
                            .unwrap_or(0);
                        let token_count: usize = by_token.map(|m| m.len()).unwrap_or(0);
                        (*pipeline, restriction_count, token_count)
                    })
                    .collect();

                let now = self.update(new_manager).await;

                for (pipeline, restriction_count, token_count) in counts {
                    let pipeline_str = pipeline.as_str();

                    gauge!(
                        "capture_event_restrictions_last_refresh_timestamp",
                        "pipeline" => pipeline_str.to_string()
                    )
                    .set(now as f64);

                    gauge!(
                        "capture_event_restrictions_loaded_count",
                        "pipeline" => pipeline_str.to_string()
                    )
                    .set(restriction_count as f64);

                    gauge!(
                        "capture_event_restrictions_tokens_count",
                        "pipeline" => pipeline_str.to_string()
                    )
                    .set(token_count as f64);

                    gauge!(
                        "capture_event_restrictions_stale",
                        "pipeline" => pipeline_str.to_string()
                    )
                    .set(0.0);
                }

                true
            }
            Err(e) => {
                error!(
                    pipelines = ?self.pipelines,
                    error = %e,
                    "Failed to refresh event restrictions, will reconnect"
                );
                false
            }
        }
    }

    /// Manually update restrictions (useful for testing). Returns the timestamp.
    pub async fn update(&self, new_manager: RestrictionManager) -> i64 {
        let mut guard = self.manager.write().await;
        *guard = new_manager;
        let now = chrono::Utc::now().timestamp();
        self.last_successful_refresh.store(now, Ordering::SeqCst);
        now
    }

    /// Whether at least one refresh has succeeded since boot. Until then the
    /// service is fail-open and every lookup returns empty, so callers that
    /// need loaded state (tests synchronizing with the initial load) poll this
    /// instead of guessing at refresh timing.
    pub fn has_loaded(&self) -> bool {
        self.last_successful_refresh.load(Ordering::SeqCst) != 0
    }

    /// Check if the cache is stale (fail-open should be active).
    fn is_stale_at(&self, now_ts: i64) -> bool {
        let last_refresh = self.last_successful_refresh.load(Ordering::SeqCst);
        if last_refresh == 0 {
            return true;
        }

        // Use saturating_sub to handle potential clock skew (if now < last_refresh, age = 0)
        let age_secs = (now_ts as u64).saturating_sub(last_refresh as u64);
        Duration::from_secs(age_secs) > self.fail_open_after
    }

    /// Get applied restrictions for an event under the given pipeline. Returns
    /// empty if fail-open is active or if the pipeline isn't in the service's
    /// allowlist (no entries are indexed for it).
    pub async fn get_restrictions(
        &self,
        token: &str,
        event: &EventContext<'_>,
        pipeline: Pipeline,
    ) -> AppliedRestrictions {
        if self.is_stale_at(event.now_ts) {
            gauge!(
                "capture_event_restrictions_stale",
                "pipeline" => pipeline.as_str().to_string()
            )
            .set(1.0);
            return AppliedRestrictions::default();
        }

        let guard = self.manager.read().await;
        let set = guard.get_restrictions(token, event, pipeline);
        AppliedRestrictions::from_restrictions(set, pipeline)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::config::CaptureMode;
    use crate::event_restrictions::repository::testing::MockRestrictionsRepository;
    use crate::event_restrictions::repository::RestrictionEntry;
    use crate::event_restrictions::types::RestrictionScope;
    use crate::v0_request::DataType;
    use common_redis::CustomRedisError;
    use strum::IntoEnumIterator;
    use tokio_util::sync::CancellationToken;

    fn test_lifecycle() -> (CancellationToken, lifecycle::Handle) {
        let shutdown_token = CancellationToken::new();
        let mut manager = lifecycle::Manager::builder("test")
            .with_trap_signals(false)
            .with_prestop_check(false)
            .with_shutdown_token(shutdown_token.clone())
            .build();
        let handle = manager.register("event-restrictions", lifecycle::ComponentOptions::new());
        let _monitor = manager.monitor_background();
        (shutdown_token, handle)
    }

    fn make_entry(token: &str, pipelines: Vec<&str>) -> RestrictionEntry {
        RestrictionEntry {
            version: Some(2),
            token: token.to_string(),
            pipelines: pipelines.into_iter().map(|s| s.to_string()).collect(),
            distinct_ids: vec![],
            session_ids: vec![],
            event_names: vec![],
            event_uuids: vec![],
            args: None,
            index: None,
        }
    }

    fn make_entry_with_filters(
        token: &str,
        pipelines: Vec<&str>,
        distinct_ids: Vec<&str>,
        event_names: Vec<&str>,
    ) -> RestrictionEntry {
        RestrictionEntry {
            version: Some(2),
            token: token.to_string(),
            pipelines: pipelines.into_iter().map(|s| s.to_string()).collect(),
            distinct_ids: distinct_ids.into_iter().map(|s| s.to_string()).collect(),
            session_ids: vec![],
            event_names: event_names.into_iter().map(|s| s.to_string()).collect(),
            event_uuids: vec![],
            args: None,
            index: None,
        }
    }

    #[tokio::test]
    async fn test_manager_applies_all_restriction_types() {
        let repo = MockRestrictionsRepository::new();

        // Set up one entry for each restriction type
        repo.set_entries(
            RestrictionType::DropEvent,
            Some(vec![make_entry("token_drop", vec!["analytics"])]),
        )
        .await;
        repo.set_entries(
            RestrictionType::ForceOverflow,
            Some(vec![make_entry("token_overflow", vec!["analytics"])]),
        )
        .await;
        repo.set_entries(
            RestrictionType::RedirectToDlq,
            Some(vec![make_entry("token_dlq", vec!["analytics"])]),
        )
        .await;
        repo.set_entries(
            RestrictionType::SkipPersonProcessing,
            Some(vec![make_entry("token_skip", vec!["analytics"])]),
        )
        .await;

        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        let event = EventContext::default();

        // Each token should have exactly its restriction type
        let drop_restrictions = manager.get_restrictions("token_drop", &event, Pipeline::Analytics);
        assert_eq!(drop_restrictions.len(), 1);
        assert!(drop_restrictions.contains(RestrictionType::DropEvent));

        let overflow_restrictions =
            manager.get_restrictions("token_overflow", &event, Pipeline::Analytics);
        assert_eq!(overflow_restrictions.len(), 1);
        assert!(overflow_restrictions.contains(RestrictionType::ForceOverflow));

        let dlq_restrictions = manager.get_restrictions("token_dlq", &event, Pipeline::Analytics);
        assert_eq!(dlq_restrictions.len(), 1);
        assert!(dlq_restrictions.contains(RestrictionType::RedirectToDlq));

        let skip_restrictions = manager.get_restrictions("token_skip", &event, Pipeline::Analytics);
        assert_eq!(skip_restrictions.len(), 1);
        assert!(skip_restrictions.contains(RestrictionType::SkipPersonProcessing));

        // Unknown token should have no restrictions
        let unknown = manager.get_restrictions("unknown_token", &event, Pipeline::Analytics);
        assert!(unknown.is_empty());
    }

    #[tokio::test]
    async fn test_manager_applies_filters_correctly() {
        let repo = MockRestrictionsRepository::new();

        repo.set_entries(
            RestrictionType::DropEvent,
            Some(vec![make_entry_with_filters(
                "token1",
                vec!["analytics"],
                vec!["user1", "user2"],
                vec!["$pageview"],
            )]),
        )
        .await;

        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        // Should match when both distinct_id and event_name match
        let event_match = EventContext {
            distinct_id: Some("user1"),
            event_name: Some("$pageview"),
            ..Default::default()
        };
        let restrictions = manager.get_restrictions("token1", &event_match, Pipeline::Analytics);
        assert!(restrictions.contains(RestrictionType::DropEvent));

        // Should NOT match when event_name doesn't match (AND logic)
        let event_wrong_name = EventContext {
            distinct_id: Some("user1"),
            event_name: Some("$identify"),
            ..Default::default()
        };
        let restrictions =
            manager.get_restrictions("token1", &event_wrong_name, Pipeline::Analytics);
        assert!(restrictions.is_empty());

        // Should NOT match when distinct_id doesn't match
        let event_wrong_user = EventContext {
            distinct_id: Some("user3"),
            event_name: Some("$pageview"),
            ..Default::default()
        };
        let restrictions =
            manager.get_restrictions("token1", &event_wrong_user, Pipeline::Analytics);
        assert!(restrictions.is_empty());
    }

    #[tokio::test]
    async fn test_manager_filters_by_pipeline() {
        let repo = MockRestrictionsRepository::new();

        repo.set_entries(
            RestrictionType::DropEvent,
            Some(vec![
                make_entry("token_analytics", vec!["analytics"]),
                make_entry("token_recordings", vec!["session_recordings"]),
                make_entry("token_both", vec!["analytics", "session_recordings"]),
            ]),
        )
        .await;

        // Fetch for analytics pipeline
        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        let event = EventContext::default();

        // analytics token should be present
        assert!(!manager
            .get_restrictions("token_analytics", &event, Pipeline::Analytics)
            .is_empty());

        // recordings token should NOT be present (entry not indexed under
        // Analytics, even though it exists under SessionRecordings — and
        // SessionRecordings isn't in the allowlist).
        assert!(manager
            .get_restrictions("token_recordings", &event, Pipeline::Analytics)
            .is_empty());

        // both token should be present
        assert!(!manager
            .get_restrictions("token_both", &event, Pipeline::Analytics)
            .is_empty());
    }

    #[tokio::test]
    async fn events_deployment_loads_restrictions_for_every_classified_pipeline() {
        for data_type in DataType::iter() {
            let Some(pipeline) = data_type.pipeline() else {
                continue;
            };
            let repo = MockRestrictionsRepository::new();
            repo.set_entries(
                RestrictionType::DropEvent,
                Some(vec![make_entry("token", vec![pipeline.as_str()])]),
            )
            .await;

            let manager = RestrictionManager::from_repository(
                &repo,
                &Pipeline::for_capture_mode(CaptureMode::Events),
            )
            .await
            .unwrap();

            let restrictions =
                manager.get_restrictions("token", &EventContext::default(), pipeline);
            assert!(
                restrictions.contains(RestrictionType::DropEvent),
                "{data_type:?}"
            );
        }
    }

    #[tokio::test]
    async fn test_manager_skips_old_version() {
        let repo = MockRestrictionsRepository::new();

        let mut old_entry = make_entry("token_old", vec!["analytics"]);
        old_entry.version = Some(1); // Old version

        let new_entry = make_entry("token_new", vec!["analytics"]);

        repo.set_entries(RestrictionType::DropEvent, Some(vec![old_entry, new_entry]))
            .await;

        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        let event = EventContext::default();

        // Old version should be skipped
        assert!(manager
            .get_restrictions("token_old", &event, Pipeline::Analytics)
            .is_empty());

        // New version should be present
        assert!(!manager
            .get_restrictions("token_new", &event, Pipeline::Analytics)
            .is_empty());
    }

    #[tokio::test]
    async fn test_manager_returns_error_when_any_fetch_fails() {
        let repo = MockRestrictionsRepository::new();

        repo.set_error(RestrictionType::DropEvent, CustomRedisError::Timeout)
            .await;
        repo.set_entries(
            RestrictionType::ForceOverflow,
            Some(vec![make_entry("token1", vec!["analytics"])]),
        )
        .await;

        let result = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics]).await;
        assert!(result.is_err());
    }

    #[tokio::test]
    async fn test_manager_handles_empty_repository() {
        let repo = MockRestrictionsRepository::new();
        // Don't set any entries

        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        assert!(manager.restrictions.is_empty());
    }

    #[tokio::test]
    async fn test_manager_multiple_restrictions_same_token() {
        let repo = MockRestrictionsRepository::new();

        // Same token has multiple restriction types
        repo.set_entries(
            RestrictionType::ForceOverflow,
            Some(vec![make_entry("token1", vec!["analytics"])]),
        )
        .await;
        repo.set_entries(
            RestrictionType::SkipPersonProcessing,
            Some(vec![make_entry("token1", vec!["analytics"])]),
        )
        .await;

        let manager = RestrictionManager::from_repository(&repo, &[Pipeline::Analytics])
            .await
            .unwrap();

        let restrictions =
            manager.get_restrictions("token1", &EventContext::default(), Pipeline::Analytics);
        assert_eq!(restrictions.len(), 2);
        assert!(restrictions.contains(RestrictionType::ForceOverflow));
        assert!(restrictions.contains(RestrictionType::SkipPersonProcessing));
    }

    // ========================================================================
    // EventRestrictionService tests
    // ========================================================================

    fn event_ctx_now() -> EventContext<'static> {
        EventContext {
            now_ts: chrono::Utc::now().timestamp(),
            ..Default::default()
        }
    }

    #[tokio::test]
    async fn test_service_is_stale_when_never_refreshed() {
        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
        // last_successful_refresh is 0, so should be stale (fail-open)
        let applied = service
            .get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.is_empty());
    }

    #[tokio::test]
    async fn test_service_returns_restrictions_after_update() {
        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));

        let mut manager = RestrictionManager::new();
        manager.insert_restrictions(
            Pipeline::Analytics,
            "token1",
            vec![Restriction {
                restriction_type: RestrictionType::DropEvent,
                scope: RestrictionScope::AllEvents,
                args: None,
            }],
        );
        service.update(manager).await;

        let applied = service
            .get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.should_drop());
    }

    #[tokio::test]
    async fn test_service_fail_open_after_timeout() {
        let service = EventRestrictionService::new(
            vec![Pipeline::Analytics],
            Duration::from_secs(1), // 1 second timeout
        );

        let mut manager = RestrictionManager::new();
        manager.insert_restrictions(
            Pipeline::Analytics,
            "token1",
            vec![Restriction {
                restriction_type: RestrictionType::DropEvent,
                scope: RestrictionScope::AllEvents,
                args: None,
            }],
        );
        service.update(manager).await;

        // Immediately after update, should return restrictions
        let applied = service
            .get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.should_drop());

        // Manually set last_successful_refresh to 10 seconds ago
        let old_timestamp = chrono::Utc::now().timestamp() - 10;
        service
            .last_successful_refresh
            .store(old_timestamp, Ordering::SeqCst);

        // Now should be stale (fail-open)
        let applied_after = service
            .get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied_after.is_empty());
    }

    // ========================================================================
    // start_refresh_task tests
    // ========================================================================

    #[tokio::test]
    async fn test_refresh_task_loads_restrictions() {
        let (shutdown_token, lifecycle_handle) = test_lifecycle();
        let shutdown_token_clone = shutdown_token.clone();

        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
        let service_clone = service.clone();

        let handle = tokio::spawn(async move {
            service_clone
                .start_refresh_task(
                    move || {
                        let st = shutdown_token_clone.clone();
                        async move {
                            st.cancel();
                            let repo = MockRestrictionsRepository::new();
                            repo.set_entries(
                                RestrictionType::DropEvent,
                                Some(vec![make_entry("token1", vec!["analytics"])]),
                            )
                            .await;
                            let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
                            Ok(result)
                        }
                    },
                    Duration::from_millis(5),
                    lifecycle_handle,
                )
                .await;
        });

        handle.await.unwrap();

        let applied = service
            .get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.should_drop());

        let unknown = service
            .get_restrictions("unknown", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(unknown.is_empty());
    }

    #[tokio::test]
    async fn test_refresh_task_retries_connection_while_fail_open() {
        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
        let (shutdown_token, lifecycle_handle) = test_lifecycle();
        let shutdown_token_clone = shutdown_token.clone();
        let attempt_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
        let count_clone = attempt_count.clone();

        let service_clone = service.clone();
        let handle = tokio::spawn(async move {
            service_clone
                .start_refresh_task(
                    move || {
                        let n = count_clone.fetch_add(1, Ordering::SeqCst);
                        let st = shutdown_token_clone.clone();
                        async move {
                            if n >= 2 {
                                st.cancel();
                            }
                            Err(CustomRedisError::Timeout)
                        }
                    },
                    Duration::from_millis(5),
                    lifecycle_handle,
                )
                .await;
        });

        handle.await.unwrap();

        let applied = service
            .get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.is_empty());
        assert!(
            attempt_count.load(Ordering::SeqCst) >= 2,
            "should have retried"
        );
    }

    #[tokio::test]
    async fn test_refresh_task_reconnects_after_refresh_failure() {
        let (shutdown_token, lifecycle_handle) = test_lifecycle();
        let shutdown_token_clone = shutdown_token.clone();
        let connect_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
        let count_clone = connect_count.clone();

        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
        let service_clone = service.clone();

        let handle = tokio::spawn(async move {
            service_clone
                .start_refresh_task(
                    move || {
                        let n = count_clone.fetch_add(1, Ordering::SeqCst);
                        let st = shutdown_token_clone.clone();
                        async move {
                            // First connection succeeds but repo returns all errors,
                            // second connection triggers shutdown so we can assert.
                            if n >= 1 {
                                st.cancel();
                            }
                            let repo = MockRestrictionsRepository::new();
                            for rt in RestrictionType::all() {
                                repo.set_error(rt, CustomRedisError::Timeout).await;
                            }
                            let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
                            Ok(result)
                        }
                    },
                    Duration::from_millis(5),
                    lifecycle_handle,
                )
                .await;
        });

        handle.await.unwrap();

        assert!(
            connect_count.load(Ordering::SeqCst) >= 2,
            "should have reconnected after refresh failure"
        );
        // Service never got a successful refresh, so it stays fail-open
        let applied = service
            .get_restrictions("token", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(applied.is_empty());
    }

    #[tokio::test]
    async fn test_refresh_task_fail_open_then_recover() {
        let (shutdown_token, lifecycle_handle) = test_lifecycle();
        let shutdown_token_clone = shutdown_token.clone();
        let attempt_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
        let count_clone = attempt_count.clone();

        let service =
            EventRestrictionService::new(vec![Pipeline::Analytics], Duration::from_secs(300));
        let service_clone = service.clone();

        let handle = tokio::spawn(async move {
            service_clone
                .start_refresh_task(
                    move || {
                        let n = count_clone.fetch_add(1, Ordering::SeqCst);
                        let st = shutdown_token_clone.clone();
                        async move {
                            if n < 2 {
                                // First two attempts fail (simulating Redis down)
                                return Err(CustomRedisError::Timeout);
                            }
                            // Third attempt succeeds with restrictions
                            st.cancel();
                            let repo = MockRestrictionsRepository::new();
                            repo.set_entries(
                                RestrictionType::ForceOverflow,
                                Some(vec![make_entry("token1", vec!["analytics"])]),
                            )
                            .await;
                            let result: Arc<dyn EventRestrictionsRepository> = Arc::new(repo);
                            Ok(result)
                        }
                    },
                    Duration::from_millis(5),
                    lifecycle_handle,
                )
                .await;
        });

        handle.await.unwrap();

        assert!(
            attempt_count.load(Ordering::SeqCst) >= 3,
            "should have attempted at least 3 times"
        );
        // After recovery, restrictions should be active
        let applied = service
            .get_restrictions("token1", &event_ctx_now(), Pipeline::Analytics)
            .await;
        assert!(
            applied.force_overflow(),
            "restrictions should be active after recovery"
        );
    }
}