Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

42.0 KB
Raw Blame
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;

use crate::billing::{BillingAggregator, FeatureFlagsLimiter, SessionReplayLimiter, UsageReporter};
use crate::cohorts::cohort_cache_manager::CohortCacheManager;
use crate::cohorts::membership::{
    CachedCohortMembershipProvider, CohortMembershipProvider, NoOpCohortMembershipProvider,
    RealtimeCohortMembershipProvider,
};
use crate::config::{Config, TeamIdCollection};
use crate::database_pools::DatabasePools;
use crate::db_monitor::DatabasePoolMonitor;
use crate::flags::flag_definitions_cache::FlagDefinitionsCache;
use crate::flags::flag_group_type_mapping::GroupTypeCacheManager;
use crate::metrics::consts::FLAG_DEFINITIONS_READS_DEDICATED_REDIS_GAUGE;
use crate::rayon_dispatcher::RayonDispatcher;
use crate::router;
use crate::tokio_monitor::TokioRuntimeMonitor;
use common_cache::NegativeCache;
use common_cookieless::CookielessManager;
use common_geoip::GeoIpClient;
use common_hypercache::{HyperCacheConfig, HyperCacheReader, S3Client};
use common_metrics::gauge;
use common_redis::{
    Client, CompressionConfig, ReadWriteClient, ReadWriteClientConfig, RedisClient,
};
use governor::clock;
use lifecycle::{ComponentOptions, Handle, LivenessHandler, Manager, ReadinessHandler};
use limiters::redis::QUOTA_LIMITER_CACHE_KEY;
use tokio::net::TcpListener;
use tokio_retry::strategy::{jitter, ExponentialBackoff};
use tokio_retry::Retry;

/// Handles for every lifecycle component this service registers, plus the readiness
/// and liveness handlers the HTTP router needs. Produced by [`register_components`]
/// and passed into [`serve`].
pub struct LifecycleHandles {
    pub http: Handle,
    pub db_monitor: Handle,
    pub cohort_cache_monitor: Handle,
    pub flag_defs_cache_monitor: Handle,
    pub tokio_monitor: Handle,
    pub readiness: ReadinessHandler,
    pub liveness: LivenessHandler,
}

/// Register the feature-flags lifecycle components and return handles for use by
/// `serve()` (and by the test harness). Call this exactly once per Manager, before
/// `manager.monitor_background()`.
///
/// `/_liveness` is hardcoded to 200 (see `lifecycle::LivenessHandler`) and no
/// component opts into `with_liveness_deadline`. For this service, if the tokio
/// runtime can serve the endpoint the process is alive — there's no meaningful
/// internal signal beyond that, and a heartbeat loop would just be checking that
/// the heartbeat itself still runs. Do not add deadlines or `report_healthy()`
/// calls here unless a real signal appears (e.g. the way capture feeds kafka
/// producer health into its handle).
pub fn register_components(manager: &mut Manager) -> LifecycleHandles {
    let http = manager.register(
        "http-server",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(30)),
    );
    let db_monitor = manager.register(
        "db-pool-monitor",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(2)),
    );
    let cohort_cache_monitor = manager.register(
        "cohort-cache-monitor",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(2)),
    );
    let flag_defs_cache_monitor = manager.register(
        "flag-definitions-cache-monitor",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(2)),
    );
    let tokio_monitor = manager.register(
        "tokio-runtime-monitor",
        ComponentOptions::new().with_graceful_shutdown(Duration::from_secs(2)),
    );
    let readiness = manager.readiness_handler();
    let liveness = manager.liveness_handler();
    LifecycleHandles {
        http,
        db_monitor,
        cohort_cache_monitor,
        flag_defs_cache_monitor,
        tokio_monitor,
        readiness,
        liveness,
    }
}

impl LifecycleHandles {
    /// Surface an init-time error to the lifecycle manager as a `Failure` on the http
    /// component (so the trigger log reads `trigger_reason="failure" reason="…"` instead
    /// of a misleading `Died { tag: "http-server" }`), and pre-complete the unstarted
    /// monitor handles so their drops don't fan out "died during shutdown" warnings in
    /// Phase 2. Call before each early `return` from `serve()`.
    fn fail_init(&self, reason: impl Into<String>) {
        self.http.signal_failure(reason);
        self.http.work_completed();
        self.db_monitor.work_completed();
        self.cohort_cache_monitor.work_completed();
        self.flag_defs_cache_monitor.work_completed();
        self.tokio_monitor.work_completed();
    }
}

pub async fn serve(
    config: Config,
    listener: TcpListener,
    rayon_dispatcher: RayonDispatcher,
    handles: LifecycleHandles,
    // Test-only override for the flags-with-cohorts reader's S3 client. `None` in
    // production; tests inject a dummy so a cache miss classifies as CacheMiss without
    // a real object store. Uses the `new_with_s3_client` testing seam on HyperCacheReader.
    flags_with_cohorts_s3: Option<Arc<dyn S3Client + Send + Sync>>,
) {
    serve_with_rate_limiter_clock(
        config,
        listener,
        rayon_dispatcher,
        handles,
        flags_with_cohorts_s3,
        clock::DefaultClock::default(),
    )
    .await;
}

pub async fn serve_with_rate_limiter_clock<C>(
    config: Config,
    listener: TcpListener,
    rayon_dispatcher: RayonDispatcher,
    handles: LifecycleHandles,
    flags_with_cohorts_s3: Option<Arc<dyn S3Client + Send + Sync>>,
    rate_limiter_clock: C,
) where
    C: clock::Clock + Clone + Send + Sync + 'static,
{
    // Configure compression based on environment variable
    let compression_config = if *config.redis_compression_enabled {
        let config = CompressionConfig::default();
        tracing::info!(
            "Redis compression enabled (threshold: {} bytes)",
            config.threshold
        );
        config
    } else {
        tracing::info!("Redis compression disabled");
        CompressionConfig::disabled()
    };

    // Create ReadWriteClient for shared Redis (non-critical path: analytics, billing, cookieless)
    // "shared" means the Redis client shares the cache with the Django PostHog web app.
    // Automatically routes reads to replica and writes to primary
    let Some(redis_client) = create_readwrite_client(
        config.get_redis_writer_url(),
        config.get_redis_reader_url(),
        "shared",
        compression_config.clone(),
        config.redis_response_timeout_ms,
        config.redis_connection_timeout_ms,
        config.redis_client_retry_count,
    )
    .await
    else {
        handles.fail_init("shared redis init failed");
        return;
    };

    // Create dedicated ReadWriteClient for flags cache (critical path isolation)
    let dedicated_redis_client =
        create_dedicated_readwrite_client(&config, compression_config.clone()).await;

    // The flags-with-cohorts reader is the one flags cache with its own cluster switch.
    let (flags_with_cohorts_redis_client, flag_definitions_cluster) =
        resolve_flag_definitions_redis_client(
            dedicated_redis_client.as_ref(),
            &redis_client,
            *config.flag_definitions_dedicated_redis_enabled,
        );

    // Per cache, not one mode string: flags_with_cohorts no longer agrees with the rest.
    let other_flags_caches = if dedicated_redis_client.is_some() {
        "dedicated"
    } else {
        "shared"
    };
    tracing::info!(
        other_flags_caches,
        flags_with_cohorts = flag_definitions_cluster.cluster(),
        flags_with_cohorts_reason = flag_definitions_cluster.reason(),
        "Feature flags Redis cluster per cache"
    );

    // Create database pools with persons routing support
    let database_pools = match DatabasePools::from_config(&config).await {
        Ok(pools) => {
            tracing::info!("Successfully created database pools");
            if config.is_persons_db_routing_enabled() {
                tracing::info!("Persons database routing is enabled");
            }
            Arc::new(pools)
        }
        Err(e) => {
            tracing::error!(
                error = %e,
                "Failed to create database pools"
            );
            handles.fail_init(format!("database pools init failed: {e}"));
            return;
        }
    };

    let geoip_service = match GeoIpClient::new(config.get_maxmind_db_path()) {
        Ok(service) => Arc::new(service),
        Err(e) => {
            tracing::error!(
                "Failed to create GeoIP service with DB path {}: {}",
                config.get_maxmind_db_path().display(),
                e
            );
            handles.fail_init(format!("geoip init failed: {e}"));
            return;
        }
    };

    let cohort_cache = Arc::new(CohortCacheManager::new(
        database_pools.non_persons_reader.clone(),
        Some(config.cohort_cache_capacity_bytes),
        Some(config.cache_ttl_seconds),
    ));

    let flag_definitions_cache = Arc::new(FlagDefinitionsCache::new(
        Some(config.flag_definitions_cache_capacity_bytes),
        Some(config.flag_definitions_cache_ttl_seconds),
    ));

    let group_type_cache = Arc::new(GroupTypeCacheManager::new(
        database_pools.persons_reader.clone(),
        Some(config.group_type_cache_max_entries),
        Some(config.group_type_cache_ttl_seconds),
    ));

    // Initialize the cohort membership provider for realtime/behavioral cohorts whenever the
    // behavioral cohorts DB pool exists: a team access row at `serving` routes a team's cohorts
    // even when REALTIME_COHORT_EVALUATION_TEAM_IDS is "none". A team that routes no cohort
    // never calls the provider, so no realtime cohort query reaches its hot path.
    let cohort_membership_provider: Arc<dyn CohortMembershipProvider> =
        if let Some(pool) = database_pools.behavioral_cohorts_reader.clone() {
            tracing::info!(
                cache_ttl_seconds = config.cohort_membership_cache_ttl_seconds,
                cache_max_entries = config.cohort_membership_cache_max_entries,
                lookup_timeout_ms = config.realtime_cohort_lookup_timeout_ms,
                "Realtime cohort evaluation enabled with behavioral cohorts DB"
            );
            let realtime = RealtimeCohortMembershipProvider::with_lookup_timeout(
                pool,
                Duration::from_millis(config.realtime_cohort_lookup_timeout_ms),
            );
            Arc::new(CachedCohortMembershipProvider::new(
                realtime,
                Some(config.cohort_membership_cache_ttl_seconds),
                Some(config.cohort_membership_cache_max_entries),
            ))
        } else {
            if config.realtime_cohort_evaluation_team_ids != TeamIdCollection::None {
                tracing::warn!(
                    "REALTIME_COHORT_EVALUATION_TEAM_IDS is set but \
                     BEHAVIORAL_COHORTS_READ_DATABASE_URL is not configured; realtime \
                     cohort lookups will treat everyone as a non-member"
                );
            }
            Arc::new(NoOpCohortMembershipProvider)
        };

    let feature_flags_billing_limiter = match FeatureFlagsLimiter::new(
        Duration::from_secs(config.billing_limiter_cache_ttl_secs),
        redis_client.clone(), // Limiter only reads from redis, ReadWriteClient automatically routes reads to replica
        QUOTA_LIMITER_CACHE_KEY.to_string(),
        None,
    ) {
        Ok(limiter) => limiter,
        Err(e) => {
            tracing::error!("Failed to create feature flags billing limiter: {}", e);
            handles.fail_init(format!("feature flags billing limiter init failed: {e}"));
            return;
        }
    };

    let session_replay_billing_limiter = match SessionReplayLimiter::new(
        Duration::from_secs(config.billing_limiter_cache_ttl_secs),
        redis_client.clone(), // Limiter only reads from redis, ReadWriteClient automatically routes reads to replica
        QUOTA_LIMITER_CACHE_KEY.to_string(),
        None,
    ) {
        Ok(limiter) => limiter,
        Err(e) => {
            tracing::error!("Failed to create session replay billing limiter: {}", e);
            handles.fail_init(format!("session replay billing limiter init failed: {e}"));
            return;
        }
    };

    let Some(redis_cookieless_client) = create_redis_client(
        &config.get_redis_cookieless_url(),
        "cookieless",
        compression_config.clone(),
        config.redis_response_timeout_ms,
        config.redis_connection_timeout_ms,
        config.redis_client_retry_count,
    )
    .await
    else {
        handles.fail_init("cookieless redis init failed");
        return;
    };

    let cookieless_manager = Arc::new(CookielessManager::new(
        config.get_cookieless_config(),
        redis_cookieless_client.clone(),
    ));

    // Create HyperCacheReader for feature flags at startup
    // This avoids per-request AWS SDK initialization overhead
    let flags_redis_client = dedicated_redis_client
        .clone()
        .unwrap_or_else(|| redis_client.clone());

    let hypercache_s3_timeout = Duration::from_millis(config.hypercache_s3_timeout_ms);

    let mut flags_hypercache_config = HyperCacheConfig::new(
        "feature_flags".to_string(),
        "flags.json".to_string(),
        config.object_storage_region.clone(),
        config.object_storage_bucket.clone(),
    );
    flags_hypercache_config.s3_timeout = hypercache_s3_timeout;
    // Etag-paired on the Django writer side, which makes HyperCacheReader refuse read repair.
    flags_hypercache_config.enable_etag = true;

    if !config.object_storage_endpoint.is_empty() {
        flags_hypercache_config.s3_endpoint = Some(config.object_storage_endpoint.clone());
    }

    let flags_hypercache_reader =
        match HyperCacheReader::new(flags_redis_client, flags_hypercache_config).await {
            Ok(reader) => {
                tracing::info!("Created HyperCacheReader for feature flags");
                Arc::new(reader)
            }
            Err(e) => {
                tracing::error!("Failed to create flags HyperCacheReader: {:?}", e);
                handles.fail_init(format!("flags hypercache init failed: {e:?}"));
                return;
            }
        };

    // Read repair for the hypercaches that are read straight through to Redis on every
    // request. Both feature_flags readers are left out: each carries a companion `:etag` key
    // that a payload-only repair would leave stale, and FlagDefinitionsCache already absorbs
    // repeat reads of a cold flags.json in process. team_metadata is left out too, below,
    // for a different reason: it gates token authentication.
    let read_repair_ttl_seconds =
        if config.hypercache_read_repair_ttl_seconds == 0 || *config.skip_writes {
            None
        } else {
            Some(config.hypercache_read_repair_ttl_seconds)
        };

    // Create HyperCacheReader for team metadata at startup
    // Uses token-based lookup instead of team_id
    let team_redis_client = dedicated_redis_client
        .clone()
        .unwrap_or_else(|| redis_client.clone());

    let mut team_hypercache_config = HyperCacheConfig::new(
        "team_metadata".to_string(),
        "full_metadata.json".to_string(),
        config.object_storage_region.clone(),
        config.object_storage_bucket.clone(),
    );
    team_hypercache_config.token_based = true;
    team_hypercache_config.s3_timeout = hypercache_s3_timeout;
    // Left out of read repair: a hit here is trusted as proof of a valid token (see
    // get_team_from_cache_or_pg), with no Postgres re-check. Team deletion clears Redis
    // before S3, so a request landing in that gap reads the deleted team from S3; repairing
    // it would resurrect that team in Redis for up to the repair TTL, instead of the single
    // stale response an unrepaired hit gives the one caller that landed in the gap.
    team_hypercache_config.read_repair_ttl_seconds = None;

    if !config.object_storage_endpoint.is_empty() {
        team_hypercache_config.s3_endpoint = Some(config.object_storage_endpoint.clone());
    }

    let team_hypercache_reader =
        match HyperCacheReader::new(team_redis_client, team_hypercache_config).await {
            Ok(reader) => {
                tracing::info!("Created HyperCacheReader for team metadata");
                Arc::new(reader)
            }
            Err(e) => {
                tracing::error!("Failed to create team HyperCacheReader: {:?}", e);
                handles.fail_init(format!("team hypercache init failed: {e:?}"));
                return;
            }
        };

    // Create HyperCacheReader for flags with cohorts (used by /flags/definitions endpoint)
    let mut flags_with_cohorts_config = HyperCacheConfig::new(
        "feature_flags".to_string(),
        "flags_with_cohorts.json".to_string(),
        config.object_storage_region.clone(),
        config.object_storage_bucket.clone(),
    );
    // Etag-paired, same as flags.json above.
    flags_with_cohorts_config.enable_etag = true;
    flags_with_cohorts_config.s3_timeout = hypercache_s3_timeout;

    if !config.object_storage_endpoint.is_empty() {
        flags_with_cohorts_config.s3_endpoint = Some(config.object_storage_endpoint.clone());
    }

    let flags_with_cohorts_hypercache_reader = match flags_with_cohorts_s3 {
        Some(s3) => Arc::new(HyperCacheReader::new_with_s3_client(
            flags_with_cohorts_redis_client,
            s3,
            flags_with_cohorts_config,
        )),
        None => {
            match HyperCacheReader::new(flags_with_cohorts_redis_client, flags_with_cohorts_config)
                .await
            {
                Ok(reader) => {
                    tracing::info!("Created HyperCacheReader for flags with cohorts");
                    Arc::new(reader)
                }
                Err(e) => {
                    tracing::error!(
                        "Failed to create flags with cohorts HyperCacheReader: {:?}",
                        e
                    );
                    handles.fail_init(format!("flags with cohorts hypercache init failed: {e:?}"));
                    return;
                }
            }
        }
    };

    // Create HyperCacheReader for remote config (array/config.json)
    // This reads the pre-computed config blob from Python's RemoteConfig.build_config()
    // Uses token-based lookup (api_token) to match Python's HyperCache key pattern
    let config_redis_client = dedicated_redis_client
        .clone()
        .unwrap_or_else(|| redis_client.clone());

    let mut config_hypercache_config = HyperCacheConfig::new(
        "array".to_string(),
        "config.json".to_string(),
        config.object_storage_region.clone(),
        config.object_storage_bucket.clone(),
    );
    config_hypercache_config.token_based = true;
    config_hypercache_config.s3_timeout = hypercache_s3_timeout;
    config_hypercache_config.read_repair_ttl_seconds = read_repair_ttl_seconds;

    if !config.object_storage_endpoint.is_empty() {
        config_hypercache_config.s3_endpoint = Some(config.object_storage_endpoint.clone());
    }

    let config_hypercache_reader =
        match HyperCacheReader::new(config_redis_client, config_hypercache_config).await {
            Ok(reader) => {
                tracing::info!("Created HyperCacheReader for remote config");
                Arc::new(reader)
            }
            Err(e) => {
                tracing::error!("Failed to create config HyperCacheReader: {:?}", e);
                handles.fail_init(format!("config hypercache init failed: {e:?}"));
                return;
            }
        };

    let team_negative_cache = NegativeCache::new(
        config.team_negative_cache_capacity,
        config.team_negative_cache_ttl_seconds,
    );
    tracing::info!(
        capacity = config.team_negative_cache_capacity,
        ttl_seconds = config.team_negative_cache_ttl_seconds,
        "Created team negative cache for invalid API tokens"
    );

    // Auth token cache: read-through cache for secret + personal API key validation.
    // Uses the flags Redis client for cache reads/writes. No in-memory negative cache —
    // Python signal handlers invalidate Redis on scope/key changes, but cannot reach
    // Rust's in-memory cache, which would cause stale denials.
    let auth_redis = dedicated_redis_client
        .clone()
        .unwrap_or_else(|| redis_client.clone());
    let auth_token_inner = Arc::new(common_cache::ReadThroughCache::new(
        auth_redis.clone(),
        auth_redis,
        common_cache::CacheConfig::with_ttl(
            crate::api::auth::TOKEN_CACHE_PREFIX,
            config.auth_token_cache_ttl_seconds,
        ),
        None,
    ));
    let auth_token_cache = Arc::new(common_cache::ReadThroughCacheWithMetrics::new(
        auth_token_inner,
        "auth",
        "token",
        &[],
    ));
    tracing::info!("Created auth token read-through cache (no negative cache)");

    if *config.skip_writes {
        tracing::warn!(
            "SKIP_WRITES is enabled: all writes to PostgreSQL and Redis are disabled. \
             This instance is running in read-only mode for safe performance testing."
        );
    }

    // Warn about deprecated environment variables
    if std::env::var("TEAM_CACHE_TTL_SECONDS").is_ok() {
        tracing::warn!(
            "TEAM_CACHE_TTL_SECONDS is deprecated and ignored. \
             Team cache TTL is now managed by Django's HyperCache."
        );
    }
    if std::env::var("FLAGS_CACHE_TTL_SECONDS").is_ok() {
        tracing::warn!(
            "FLAGS_CACHE_TTL_SECONDS is deprecated and ignored. \
             Flags cache TTL is now managed by Django's HyperCache."
        );
    }

    let service_mode = config.service_mode.clone();

    // Spawn monitor tasks only after all fallible init succeeds. If an early return
    // fired above, the manager has already seen the real `Failure` reason and the
    // unstarted monitors are pre-completed.
    let LifecycleHandles {
        http: http_handle,
        db_monitor: db_monitor_handle,
        cohort_cache_monitor: cohort_cache_monitor_handle,
        flag_defs_cache_monitor: flag_defs_cache_monitor_handle,
        tokio_monitor: tokio_monitor_handle,
        readiness,
        liveness,
    } = handles;

    let db_monitor = DatabasePoolMonitor::new(database_pools.clone(), &config);
    tokio::spawn(async move {
        db_monitor.start_monitoring(db_monitor_handle).await;
    });

    let cohort_cache_clone = cohort_cache.clone();
    let cohort_cache_monitor_interval = config.cohort_cache_monitor_interval_secs;
    tokio::spawn(async move {
        cohort_cache_clone
            .start_monitoring(cohort_cache_monitor_interval, cohort_cache_monitor_handle)
            .await;
    });

    let flag_defs_cache_clone = flag_definitions_cache.clone();
    let flag_defs_cache_monitor_interval = config.flag_definitions_cache_monitor_interval_secs;
    tokio::spawn(async move {
        flag_defs_cache_clone
            .start_monitoring(
                flag_defs_cache_monitor_interval,
                flag_defs_cache_monitor_handle,
            )
            .await;
    });

    let tokio_monitor = TokioRuntimeMonitor::new(&tokio::runtime::Handle::current());
    tokio::spawn(async move {
        tokio_monitor.start_monitoring(tokio_monitor_handle).await;
    });

    let usage_reporter = UsageReporter::new(
        &config.usage_ingestion_addr,
        config.usage_ingestion_tls,
        config.usage_ingestion_teams.clone(),
        config.usage_ingestion_timeout_ms,
    )
    .expect("invalid usage-ingestion configuration");
    let billing_aggregator: Arc<BillingAggregator> = BillingAggregator::start_with_usage_reporter(
        redis_client.clone(),
        config.get_billing_aggregator_config(),
        usage_reporter,
    );

    let app = router::router_with_rate_limiter_clock(
        redis_client,
        dedicated_redis_client,
        database_pools,
        cohort_cache,
        group_type_cache,
        geoip_service,
        readiness,
        liveness,
        feature_flags_billing_limiter,
        session_replay_billing_limiter,
        cookieless_manager,
        flags_hypercache_reader,
        flag_definitions_cache,
        flags_with_cohorts_hypercache_reader,
        team_hypercache_reader,
        config_hypercache_reader,
        rayon_dispatcher,
        team_negative_cache,
        auth_token_cache,
        cohort_membership_provider,
        billing_aggregator.clone(),
        config,
        rate_limiter_clock,
    );

    // Emitted here, not where the cluster is resolved: `router_with_rate_limiter_clock` installs
    // the Prometheus recorder, and a gauge set before that goes to the no-op recorder.
    gauge(
        FLAG_DEFINITIONS_READS_DEDICATED_REDIS_GAUGE,
        &[(
            "reason".to_string(),
            flag_definitions_cluster.reason().to_string(),
        )],
        if flag_definitions_cluster.reads_dedicated() {
            1.0
        } else {
            0.0
        },
    );

    tracing::info!(
        service_mode = ?service_mode,
        "listening on {:?}",
        listener.local_addr().unwrap()
    );

    // Always signal HTTP completion (success or failure) so the manager's
    // background monitor can complete shutdown. Skipping either arm would wedge
    // `monitor_guard.wait()` in the caller.
    let serve_result = axum::serve(
        listener,
        app.into_make_service_with_connect_info::<SocketAddr>(),
    )
    .with_graceful_shutdown(http_handle.shutdown_signal())
    .await;

    // Must run *after* `axum::serve(...).await` resolves so axum has drained
    // in-flight requests; flushing earlier would miss late-arriving records.
    billing_aggregator.shutdown().await;

    match serve_result {
        Ok(()) => http_handle.work_completed(),
        Err(e) => {
            tracing::error!("HTTP server error: {e}");
            http_handle.signal_failure(e.to_string());
            // Mark completed so HandleInner::Drop emits WorkCompleted instead of
            // racing the manager's processing of the Failure event and emitting
            // a duplicate Died (which oncall would read as a second failure).
            http_handle.work_completed();
        }
    }
}

/// Create a ReadWriteClient that automatically routes reads to replica and writes to primary
///
/// Returns None and logs error if client creation fails after all retries
async fn create_readwrite_client(
    writer_url: &str,
    reader_url: &str,
    client_type: &str,
    compression_config: CompressionConfig,
    response_timeout_ms: u64,
    connection_timeout_ms: u64,
    retry_count: u32,
) -> Option<Arc<dyn Client + Send + Sync>> {
    let rw_config = ReadWriteClientConfig::new(
        writer_url.to_string(),
        reader_url.to_string(),
        compression_config,
        common_redis::RedisValueFormat::default(),
        if response_timeout_ms == 0 {
            None
        } else {
            Some(Duration::from_millis(response_timeout_ms))
        },
        if connection_timeout_ms == 0 {
            None
        } else {
            Some(Duration::from_millis(connection_timeout_ms))
        },
    );

    // Use exponential backoff with jitter: 100ms, 200ms, 400ms, etc.
    let retry_strategy = ExponentialBackoff::from_millis(100)
        .map(jitter)
        .take(retry_count as usize);

    let client_type_owned = client_type.to_string();

    let result = Retry::spawn(retry_strategy, || async {
        match ReadWriteClient::with_config(rw_config.clone()).await {
            Ok(client) => Ok(Ok(client)),
            Err(e) => {
                if e.is_unrecoverable_error() {
                    tracing::error!(
                        "Permanent error creating {} ReadWriteClient: {}",
                        client_type_owned,
                        e
                    );
                    Ok(Err(e))
                } else {
                    tracing::debug!(
                        "Transient error creating {} ReadWriteClient, will retry: {}",
                        client_type_owned,
                        e
                    );
                    Err(e)
                }
            }
        }
    })
    .await;

    match result {
        Ok(Ok(client)) => {
            tracing::info!(
                "Created {} ReadWriteClient (writer: {}, reader: {})",
                client_type,
                writer_url,
                reader_url
            );
            Some(Arc::new(client))
        }
        Ok(Err(_)) => None,
        Err(e) => {
            tracing::error!(
                "Failed to create {} ReadWriteClient after {} retries: {}",
                client_type_owned,
                retry_count,
                e
            );
            None
        }
    }
}

/// Which cluster the `/flags/definitions` reader resolved to. `NoDedicatedClient` is kept
/// distinct from `Disabled` because both read shared, but only one of them is a fault.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FlagDefinitionsCluster {
    Disabled,
    Dedicated,
    NoDedicatedClient,
}

impl FlagDefinitionsCluster {
    /// The cluster the reader serves from. Two of the three outcomes read shared, so this
    /// answers "which cluster" where `reason` answers "why".
    fn cluster(self) -> &'static str {
        match self {
            Self::Dedicated => "dedicated",
            Self::Disabled | Self::NoDedicatedClient => "shared",
        }
    }

    fn reason(self) -> &'static str {
        match self {
            Self::Disabled => "disabled",
            Self::Dedicated => "dedicated",
            Self::NoDedicatedClient => "no_dedicated_client",
        }
    }

    fn reads_dedicated(self) -> bool {
        matches!(self, Self::Dedicated)
    }
}

/// Pick the Redis cluster the `/flags/definitions` readers use.
///
/// Do not give the ETag a gate of its own. Both the payload and the ETag come from this one
/// client, so a stale-but-present ETag on one cluster can never match a client's `If-None-Match`
/// while the other cluster holds a newer payload. That pairing answers 304 and pins the SDK to
/// stale definitions with no error on any metric.
fn resolve_flag_definitions_redis_client(
    dedicated_redis_client: Option<&Arc<dyn Client + Send + Sync>>,
    shared_redis_client: &Arc<dyn Client + Send + Sync>,
    dedicated_enabled: bool,
) -> (Arc<dyn Client + Send + Sync>, FlagDefinitionsCluster) {
    match (dedicated_enabled, dedicated_redis_client) {
        (true, Some(dedicated)) => (dedicated.clone(), FlagDefinitionsCluster::Dedicated),
        (true, None) => {
            tracing::warn!(
                "FLAG_DEFINITIONS_DEDICATED_REDIS_ENABLED is set but no dedicated flags Redis \
                 client exists. /flags/definitions keeps reading from shared Redis."
            );
            (
                shared_redis_client.clone(),
                FlagDefinitionsCluster::NoDedicatedClient,
            )
        }
        (false, _) => (
            shared_redis_client.clone(),
            FlagDefinitionsCluster::Disabled,
        ),
    }
}

/// Create dedicated ReadWriteClient for flags cache with graceful fallback
///
/// Implements fallback strategy:
/// 1. FLAGS_REDIS_READER_URL is not set → use FLAGS_REDIS_URL for both reads and writes
/// 2. FLAGS_REDIS_URL is not set → return None (use shared Redis)
///
/// Returns: Optional ReadWriteClient
async fn create_dedicated_readwrite_client(
    config: &Config,
    compression_config: CompressionConfig,
) -> Option<Arc<dyn Client + Send + Sync>> {
    let writer_url = config.get_flags_redis_writer_url();
    let reader_url = config.get_flags_redis_reader_url();

    match (writer_url, reader_url) {
        (Some(w_url), Some(r_url)) => {
            tracing::info!(
                "Creating dedicated flags ReadWriteClient with separate reader endpoint"
            );
            create_readwrite_client(
                w_url,
                r_url,
                "dedicated flags",
                compression_config,
                config.redis_response_timeout_ms,
                config.redis_connection_timeout_ms,
                config.redis_client_retry_count,
            )
            .await
        }
        (Some(w_url), None) => {
            tracing::info!("Creating dedicated flags ReadWriteClient using writer URL for both reads and writes");
            create_readwrite_client(
                w_url,
                w_url,
                "dedicated flags",
                compression_config,
                config.redis_response_timeout_ms,
                config.redis_connection_timeout_ms,
                config.redis_client_retry_count,
            )
            .await
        }
        (None, Some(_)) => {
            tracing::warn!(
                "FLAGS_REDIS_READER_URL set but FLAGS_REDIS_URL not set. \
                 Cannot use reader without writer. Falling back to shared Redis."
            );
            None
        }
        (None, None) => {
            tracing::info!(
                "Using shared Redis for flags cache (no dedicated flags Redis configured)"
            );
            None
        }
    }
}

/// Helper to create a Redis client with error logging and retry logic
/// Returns None and logs an error if client creation fails after all retries
///
/// Retry behavior:
/// - Delegates to redis crate's `is_unrecoverable_error()` for error classification
/// - Overrides: InvalidClientConfig and AuthenticationFailed always treated as permanent (no retry)
/// - In practice, most connection errors (DNS, connection refused) are also permanent
/// - Uses exponential backoff with jitter when retrying transient errors
pub async fn create_redis_client(
    url: &str,
    client_type: &str,
    compression_config: CompressionConfig,
    response_timeout_ms: u64,
    connection_timeout_ms: u64,
    retry_count: u32,
) -> Option<Arc<RedisClient>> {
    // Use exponential backoff with jitter: 100ms, 200ms, 400ms, etc.
    // When retry_count=0, .take(0) means no retries (only initial attempt)
    let retry_strategy = ExponentialBackoff::from_millis(100)
        .map(jitter)
        .take(retry_count as usize);

    let url_owned = url.to_string();
    let client_type_owned = client_type.to_string();

    let result = Retry::spawn(retry_strategy, || async {
        match RedisClient::with_config(
            url_owned.clone(),
            compression_config.clone(),
            common_redis::RedisValueFormat::default(),
            if response_timeout_ms == 0 {
                None
            } else {
                Some(Duration::from_millis(response_timeout_ms))
            },
            if connection_timeout_ms == 0 {
                None
            } else {
                Some(Duration::from_millis(connection_timeout_ms))
            },
        )
        .await
        {
            Ok(client) => Ok(Ok(client)),
            Err(e) => {
                if e.is_unrecoverable_error() {
                    tracing::error!(
                        "Permanent error creating {} Redis client for URL {}: {}",
                        client_type_owned,
                        url_owned,
                        e
                    );
                    // Return Ok(Err) to stop retrying but signal error
                    Ok(Err(e))
                } else {
                    tracing::debug!(
                        "Transient error creating {} Redis client, will retry: {}",
                        client_type_owned,
                        e
                    );
                    Err(e)
                }
            }
        }
    })
    .await;

    // Use nested Result to distinguish:
    // - Ok(Ok(client)): successful connection
    // - Ok(Err(e)): permanent error, don't retry
    // - Err(e): retryable error that will trigger retry logic
    match result {
        Ok(Ok(client)) => Some(Arc::new(client)),
        Ok(Err(_)) => {
            // Permanent error - already logged above
            None
        }
        Err(e) => {
            tracing::error!(
                "Failed to create {} Redis client for URL {} after {} retries: {}",
                client_type_owned,
                url_owned,
                retry_count,
                e
            );
            None
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use common_redis::MockRedisClient;
    use lifecycle::LifecycleError;
    use rstest::rstest;

    /// Locks in the contract that drives the early-return paths in `serve()`:
    /// `fail_init` must surface a `ComponentFailure { tag: "http-server", reason }` —
    /// not a misleading `ComponentDied { tag: "http-server" }` from the http handle's
    /// drop, which is the easy regression mode if a future component is added to
    /// `LifecycleHandles` without being wired into `fail_init`.
    #[tokio::test]
    async fn fail_init_reports_failure_not_died() {
        let mut manager = Manager::builder("test")
            .with_trap_signals(false)
            .with_prestop_check(false)
            .with_global_shutdown_timeout(Duration::from_secs(5))
            .build();
        let handles = register_components(&mut manager);
        let guard = manager.monitor_background();

        handles.fail_init("redis init failed");
        drop(handles);

        let result = tokio::time::timeout(Duration::from_secs(3), guard.wait())
            .await
            .expect("monitor did not finish within timeout");
        assert!(
            matches!(
                &result,
                Err(LifecycleError::ComponentFailure { tag, reason })
                    if tag == "http-server" && reason == "redis init failed"
            ),
            "expected ComponentFailure {{ tag: \"http-server\", reason: \"redis init failed\" }}, got {result:?}"
        );
    }

    /// An inverted toggle here moves a production read path with no other signal.
    #[rstest]
    #[case::off_with_dedicated(false, true, FlagDefinitionsCluster::Disabled, "disabled", "shared")]
    #[case::off_without_dedicated(
        false,
        false,
        FlagDefinitionsCluster::Disabled,
        "disabled",
        "shared"
    )]
    #[case::on_with_dedicated(
        true,
        true,
        FlagDefinitionsCluster::Dedicated,
        "dedicated",
        "dedicated"
    )]
    #[case::on_without_dedicated(
        true,
        false,
        FlagDefinitionsCluster::NoDedicatedClient,
        "no_dedicated_client",
        "shared"
    )]
    fn test_resolve_flag_definitions_redis_client(
        #[case] dedicated_enabled: bool,
        #[case] dedicated_present: bool,
        #[case] expected_cluster: FlagDefinitionsCluster,
        #[case] expected_reason: &str,
        #[case] expected_redis: &str,
    ) {
        let shared: Arc<dyn Client + Send + Sync> = Arc::new(MockRedisClient::new());
        let dedicated: Arc<dyn Client + Send + Sync> = Arc::new(MockRedisClient::new());

        let (resolved, cluster) = resolve_flag_definitions_redis_client(
            dedicated_present.then_some(&dedicated),
            &shared,
            dedicated_enabled,
        );

        let expected = if expected_cluster.reads_dedicated() {
            &dedicated
        } else {
            &shared
        };
        assert!(Arc::ptr_eq(&resolved, expected));
        assert_eq!(cluster, expected_cluster);
        // Runbook step 2 dispatches the on-call on these two strings, so a rename here
        // sends them to the wrong Redis endpoint during the cutover.
        assert_eq!(cluster.reason(), expected_reason);
        assert_eq!(cluster.cluster(), expected_redis);
    }

    #[tokio::test]
    async fn test_create_dedicated_readwrite_client_no_config() {
        // When FLAGS_REDIS_URL is not set, should return None
        let config = Config {
            flags_redis_url: "".to_string(),
            flags_redis_reader_url: "".to_string(),
            redis_client_retry_count: 0,
            ..Config::default_test_config()
        };

        let compression_config = CompressionConfig::disabled();
        let client = create_dedicated_readwrite_client(&config, compression_config).await;

        assert!(client.is_none());
    }

    #[tokio::test]
    async fn test_create_dedicated_readwrite_client_with_invalid_url() {
        // When FLAGS_REDIS_URL is set but unreachable, should return None
        let config = Config {
            flags_redis_url: "redis://invalid-host:6379/".to_string(),
            flags_redis_reader_url: "".to_string(),
            redis_client_retry_count: 0, // No retries for fast test
            ..Config::default_test_config()
        };

        let compression_config = CompressionConfig::disabled();
        let client = create_dedicated_readwrite_client(&config, compression_config).await;

        assert!(client.is_none());
    }

    #[tokio::test]
    async fn test_create_dedicated_readwrite_client_reader_without_writer() {
        // When only FLAGS_REDIS_READER_URL is set (misconfiguration), should return None
        let config = Config {
            flags_redis_url: "".to_string(),
            flags_redis_reader_url: "redis://localhost:6379/".to_string(),
            redis_client_retry_count: 0,
            ..Config::default_test_config()
        };

        let compression_config = CompressionConfig::disabled();
        let client = create_dedicated_readwrite_client(&config, compression_config).await;

        assert!(client.is_none());
    }

    #[tokio::test]
    async fn test_redis_error_types() {
        use common_redis::RedisClient;

        let test_cases = vec![
            ("absolutegarbage", "Garbage URL"),
            ("wrong-protocol://localhost:6379", "Wrong protocol"),
            ("redis://localhost:6378", "Wrong port (connection refused)"),
        ];

        for (url, description) in test_cases {
            println!("\n--- Testing: {description} ---");
            println!("URL: {url}");

            let result = RedisClient::with_config(
                url.to_string(),
                CompressionConfig::disabled(),
                common_redis::RedisValueFormat::default(),
                Some(Duration::from_millis(100)),
                Some(Duration::from_millis(5000)),
            )
            .await;

            match result {
                Ok(_) => println!("✅ Success (unexpected!)"),
                Err(e) => {
                    println!("❌ Error type: {e:?}");
                    println!("   Error message: {e}");
                    println!("   Is recoverable: {}", !e.is_unrecoverable_error());
                }
            }
        }
    }
}