Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

85.4 KB
Raw Blame
//! Multi-tier cache reader for PostHog, matching Django's HyperCache behavior.
//!
//! Reads from Redis (primary) then S3 (fallback) for flag definitions and similar data.
//!
//! ```rust,no_run
//! use common_hypercache::{HyperCacheConfig, HyperCacheReader, KeyType};
//! use std::sync::Arc;
//!
//! # #[tokio::main]
//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
//! // Configure the cache
//! let config = HyperCacheConfig::new(
//!     "flags".to_string(),
//!     "definitions".to_string(),
//!     "us-east-1".to_string(),
//!     "my-bucket".to_string()
//! );
//!
//! // Create the cache reader (requires Redis client)
//! # let redis_client = Arc::new(common_redis::MockRedisClient::new());
//! let reader = HyperCacheReader::new(redis_client, config).await?;
//!
//! // Get data from cache (tries Redis first, then S3)
//! let team_key = KeyType::string("team-123");
//! let data = reader.get(&team_key).await?;
//! # Ok(())
//! # }
//! ```

pub mod writer;

use anyhow::Result;
use aws_config::BehaviorVersion;
use aws_sdk_s3::Client as AwsS3SdkClient;
use common_metrics::{histogram, inc};
use common_redis::Client as RedisClient;
#[cfg(all(test, feature = "mock-client"))]
use common_s3::MockS3Client;
use common_s3::S3Impl;
// Re-exported so callers of the public `new_with_s3_client` can name the S3 trait
// (e.g. to inject a test double) without depending on common_s3 directly.
pub use common_s3::{S3Client, S3Error};
use common_types::{TeamId, TeamIdentifier};
#[cfg(all(test, feature = "mock-client"))]
use mockall::predicate;
use serde::de::DeserializeOwned;
use serde_json::Value;
use std::fmt::{self, Display};
use std::sync::Arc;
use std::time::{Duration, Instant};
use thiserror::Error;
use tokio::time::timeout;
use tracing::{debug, warn};

/// Metric name for tracking hypercache operations in Prometheus (same one used in Django's HyperCache)
pub const HYPERCACHE_COUNTER_NAME: &str = "posthog_hypercache_get_from_cache";

/// Metric name for tracking Redis failure reasons (timeout, get_error, pickle_error, json_error)
const HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME: &str = "posthog_hypercache_redis_miss_reason";

/// Metric name for tracking read-repair writes back into Redis after an S3 hit.
/// Labels: `namespace`, `value`, `result` (success | skipped | error), where `skipped`
/// means the key already existed and the repair deferred to it.
const HYPERCACHE_READ_REPAIR_COUNTER_NAME: &str = "posthog_hypercache_read_repair";

/// Per-tier latency histogram for the Redis read inside `get_typed_with_source`.
/// Labels: `namespace`, `value`, `outcome` (hit | miss | timeout | error).
pub const HYPERCACHE_REDIS_OP_MS: &str = "posthog_hypercache_redis_op_ms";

/// Per-tier latency histogram for the S3 read inside `get_typed_with_source`.
/// Labels: `namespace`, `value`, `outcome` (hit | miss | timeout | error).
pub const HYPERCACHE_S3_OP_MS: &str = "posthog_hypercache_s3_op_ms";

/// Tombstone metric for tracking "impossible" failures that should never happen in production.
/// This is duplicated from feature_flags::metrics::consts::TOMBSTONE_COUNTER because hypercache
/// is a standalone crate that shouldn't depend on feature-flags.
const TOMBSTONE_COUNTER_NAME: &str = "posthog_tombstone_total";

/// Sentinel value used in Redis to indicate that a cache key exists but has no data.
/// This value is written by Django's HyperCache when a team has no flags.
pub const HYPER_CACHE_EMPTY_VALUE: &str = "__missing__";

/// Suffix appended to a hypercache key to address its companion ETag entry. Mirrors
/// Django's `HyperCache.get_etag_key` (`{cache_key}:etag`). Shared with
/// `HyperCacheWriter` so the read and write paths can never disagree on the layout.
pub(crate) const ETAG_KEY_SUFFIX: &str = ":etag";

/// Cache key type matching Django's KeyType = Team | str | int
#[derive(Debug)]
pub enum KeyType {
    Team(Box<dyn TeamIdentifier>),
    String(String),
    Int(TeamId),
}

impl KeyType {
    pub fn team(team: impl TeamIdentifier + 'static) -> Self {
        KeyType::Team(Box::new(team))
    }

    pub fn string(s: impl Into<String>) -> Self {
        KeyType::String(s.into())
    }

    pub fn int(id: TeamId) -> Self {
        KeyType::Int(id)
    }
}

impl From<&str> for KeyType {
    fn from(s: &str) -> Self {
        KeyType::String(s.to_string())
    }
}

impl From<String> for KeyType {
    fn from(s: String) -> Self {
        KeyType::String(s)
    }
}

impl From<TeamId> for KeyType {
    fn from(id: TeamId) -> Self {
        KeyType::Int(id)
    }
}

// No blanket From<T: TeamIdentifier> to avoid conflicts
impl From<common_types::Team> for KeyType {
    fn from(team: common_types::Team) -> Self {
        KeyType::Team(Box::new(team))
    }
}

impl Display for KeyType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            KeyType::Team(team) => write!(
                f,
                "Team(id: {}, token: {})",
                team.team_id(),
                team.api_token()
            ),
            KeyType::String(s) => write!(f, "String({s})"),
            KeyType::Int(i) => write!(f, "Int({i})"),
        }
    }
}

#[derive(Error, Debug)]
pub enum HyperCacheError {
    #[error("Redis error: {0}")]
    Redis(#[from] common_redis::CustomRedisError),

    #[error("S3 error: {0}")]
    S3(#[from] S3Error),

    #[error("JSON parsing error: {0}")]
    Json(#[from] serde_json::Error),

    #[error("Pickle deserialization error: {0}")]
    Pickle(String),

    #[error("Cache miss - data not found in any tier")]
    CacheMiss,

    #[error("Timeout error: {0}")]
    Timeout(String),
}

/// Raw JSON string before deserialization, or the `__missing__` sentinel.
enum RawJsonResult {
    Json(String),
    Empty,
}

#[derive(Debug, Clone, PartialEq)]
pub enum CacheSource {
    Redis,
    /// S3 answered after Redis confirmed the key was absent, so the entry is genuinely
    /// missing from the Redis tier.
    S3,
    /// S3 answered after the Redis read failed, timed out, or returned a value that would
    /// not decode. Whether the key exists in Redis is unknown. Callers that repair or
    /// rebuild the Redis tier must act on `S3` only, because acting on this variant would
    /// write during a Redis incident, when the cluster can least afford it. Canonical logs
    /// group both variants as "s3".
    S3AfterRedisError,
    Fallback,
}

impl CacheSource {
    /// Returns a consistent string representation for canonical logging.
    pub fn as_log_str(&self) -> &'static str {
        match self {
            CacheSource::Redis => "redis",
            CacheSource::S3 => "s3",
            CacheSource::S3AfterRedisError => "s3",
            CacheSource::Fallback => "fallback",
        }
    }
}

#[derive(Debug, Clone)]
pub struct HyperCacheConfig {
    pub s3_bucket: String,
    pub s3_region: String,
    pub s3_endpoint: Option<String>,
    pub redis_timeout: Duration,
    pub s3_timeout: Duration,
    pub namespace: String,
    pub object_name: String,
    pub token_based: bool,
    pub enable_etag: bool,
    pub django_cache_version: String,
    /// When set, `HyperCacheWriter::set` records each successful write into this Redis
    /// sorted set with an expiry-timestamp score, mirroring Python's
    /// `HyperCache._track_expiry`. `None` disables expiry tracking.
    pub expiry_sorted_set_key: Option<String>,
    /// When set, an S3 hit that followed a Redis miss writes the payload back into Redis
    /// with this TTL, so the next reader for the same key is served by Redis. `None`
    /// (the default) leaves the reader read-only. Not supported on etag-enabled
    /// namespaces: `HyperCacheReader` construction warns and disables it there. A value
    /// above `HyperCacheReader::MAX_READ_REPAIR_TTL_SECONDS` is capped there too.
    ///
    /// Keep this short. It is a stampede damper for cold keys, not a substitute for the
    /// writer: repaired entries are deliberately not registered in `expiry_sorted_set_key`,
    /// so Django's refresh job stays the only thing that owns an entry's real lifetime.
    /// Whether a repair can resurrect a key deleted by `HyperCacheWriter::delete` is fixed
    /// at read time by whether the S3 read preceded the delete; the TTL doesn't affect that.
    /// What the TTL bounds is how long a resurrected entry lingers before it expires.
    pub read_repair_ttl_seconds: Option<u64>,
}

impl HyperCacheConfig {
    /// Create config with explicit settings (defaults django_cache_version to "1")
    pub fn new(
        namespace: String,
        object_name: String,
        s3_region: String,
        s3_bucket: String,
    ) -> Self {
        Self {
            s3_bucket,
            s3_region,
            s3_endpoint: None,
            redis_timeout: Duration::from_millis(500),
            s3_timeout: Duration::from_secs(3),
            namespace,
            object_name,
            token_based: false,
            enable_etag: false,
            django_cache_version: "1".to_string(),
            expiry_sorted_set_key: None,
            read_repair_ttl_seconds: None,
        }
    }

    /// Create config with custom django cache version
    pub fn with_django_cache_version(
        namespace: String,
        object_name: String,
        s3_region: String,
        s3_bucket: String,
        django_cache_version: String,
    ) -> Self {
        Self {
            s3_bucket,
            s3_region,
            s3_endpoint: None,
            redis_timeout: Duration::from_millis(500),
            s3_timeout: Duration::from_secs(3),
            namespace,
            object_name,
            token_based: false,
            enable_etag: false,
            django_cache_version,
            expiry_sorted_set_key: None,
            read_repair_ttl_seconds: None,
        }
    }

    /// Generate cache key for Redis (includes Django's posthog:version: prefix)
    pub fn get_redis_cache_key(&self, key: &KeyType) -> String {
        let base_key = self.get_base_cache_key(key);
        format!("posthog:{}:{}", self.django_cache_version, base_key)
    }

    /// Generate cache key for S3 (no prefix, matches Django's object_storage keys)
    pub fn get_s3_cache_key(&self, key: &KeyType) -> String {
        self.get_base_cache_key(key)
    }

    /// Mirror Python's `HyperCache.get_cache_identifier`: api_token for token-based
    /// caches, team_id as string otherwise. Used for both key generation and
    /// expiry-tracking sorted-set membership.
    pub fn get_cache_identifier(&self, key: &KeyType) -> String {
        if self.token_based {
            match key {
                KeyType::Team(team) => team.api_token().to_string(),
                KeyType::String(s) => s.clone(),
                KeyType::Int(i) => i.to_string(),
            }
        } else {
            match key {
                KeyType::Team(team) => team.team_id().to_string(),
                KeyType::String(s) => s.clone(),
                KeyType::Int(i) => i.to_string(),
            }
        }
    }

    /// Generate base cache key (used by both Redis and S3, but Redis adds prefix)
    fn get_base_cache_key(&self, key: &KeyType) -> String {
        let key_str = self.get_cache_identifier(key);
        let scope = if self.token_based {
            "team_tokens"
        } else {
            "teams"
        };
        format!(
            "cache/{}/{}/{}/{}",
            scope, key_str, self.namespace, self.object_name
        )
    }
}

pub struct HyperCacheReader {
    redis_client: Arc<dyn RedisClient + Send + Sync>,
    s3_client: Arc<dyn S3Client + Send + Sync>,
    config: HyperCacheConfig,
}

impl HyperCacheReader {
    /// Read repair is a stampede damper, not a writer. A TTL beyond this would outlive the
    /// refresh cycle and widen how long a `HyperCacheWriter::delete` race can resurrect a
    /// deleted key for. Caps a misconfigured env var (e.g. seconds vs. milliseconds) instead
    /// of letting it silently disable the short-TTL invariant.
    const MAX_READ_REPAIR_TTL_SECONDS: u64 = 3_600; // 1 hour

    pub async fn new(
        redis_client: Arc<dyn RedisClient + Send + Sync>,
        config: HyperCacheConfig,
    ) -> Result<Self> {
        let mut aws_config_builder = aws_config::defaults(BehaviorVersion::latest())
            .region(aws_config::Region::new(config.s3_region.clone()));

        if let Some(endpoint) = &config.s3_endpoint {
            aws_config_builder = aws_config_builder.endpoint_url(endpoint);
        }

        let aws_config = aws_config_builder.load().await;

        let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&aws_config);
        if config.s3_endpoint.is_some() {
            s3_config_builder = s3_config_builder.force_path_style(true);
        }

        let aws_s3_client = AwsS3SdkClient::from_conf(s3_config_builder.build());
        let s3_client = Arc::new(S3Impl::new(aws_s3_client)) as Arc<dyn S3Client + Send + Sync>;

        Ok(Self::new_with_s3_client(redis_client, s3_client, config))
    }

    /// Create a new HyperCacheReader with a custom S3 client (useful for testing)
    pub fn new_with_s3_client(
        redis_client: Arc<dyn RedisClient + Send + Sync>,
        s3_client: Arc<dyn S3Client + Send + Sync>,
        mut config: HyperCacheConfig,
    ) -> Self {
        // Read repair writes only the payload, but an etag-enabled namespace needs the
        // payload and its companion etag written atomically (see
        // `HyperCacheWriter::set_with_etag`). Repairing only the payload would leave the
        // pair inconsistent, so those namespaces are left to the writer. The refusal is
        // announced at construction rather than silently skipping every repair.
        if config.enable_etag && config.read_repair_ttl_seconds.is_some() {
            warn!(
                namespace = %config.namespace,
                value = %config.object_name,
                "read repair is not supported for etag-enabled namespaces; disabling it for this reader"
            );
            config.read_repair_ttl_seconds = None;
        }

        if let Some(ttl) = config.read_repair_ttl_seconds {
            if ttl > Self::MAX_READ_REPAIR_TTL_SECONDS {
                warn!(
                    namespace = %config.namespace,
                    value = %config.object_name,
                    configured_ttl_seconds = ttl,
                    max_ttl_seconds = Self::MAX_READ_REPAIR_TTL_SECONDS,
                    "read repair TTL exceeds maximum; capping at maximum"
                );
                config.read_repair_ttl_seconds = Some(Self::MAX_READ_REPAIR_TTL_SECONDS);
            }
        }

        Self {
            redis_client,
            s3_client,
            config,
        }
    }

    pub async fn get_with_source(
        &self,
        key: &KeyType,
    ) -> Result<(Value, CacheSource), HyperCacheError> {
        let (data, source) = self.get_typed_with_source::<Value>(key).await?;
        Ok((data.unwrap_or(Value::Null), source))
    }

    pub async fn get(&self, key: &KeyType) -> Result<Value, HyperCacheError> {
        let (data, _source) = self.get_with_source(key).await?;
        Ok(data)
    }

    /// Get a value from cache with fallback support
    ///
    /// This method tries to get data from cache (Redis first, then S3), and if both
    /// cache tiers miss, calls the provided fallback function to retrieve the data
    /// from an alternative source (e.g., database, API, computation, etc.).
    ///
    /// Unlike a read-through cache pattern, this method does NOT write the fallback
    /// result back to the cache. This is intentional to handle catastrophic cache
    /// miss scenarios without potentially corrupting the cache with data that may
    /// not match the expected format or freshness requirements.
    ///
    /// # Arguments
    /// * `key` - The key to look up
    /// * `fallback` - Function to call if both cache tiers miss
    ///
    /// # Returns
    /// * `Ok((Value, CacheSource))` - The value and its source (Redis, S3, or Fallback)
    /// * `Err(E)` - Error from the fallback function, or HyperCacheError if fallback returns None
    pub async fn get_with_source_or_fallback<F, Fut, E>(
        &self,
        key: &KeyType,
        fallback: F,
    ) -> Result<(Value, CacheSource), E>
    where
        F: FnOnce() -> Fut,
        Fut: std::future::Future<Output = Result<Option<Value>, E>>,
        E: From<HyperCacheError>,
    {
        let (data, source) = self
            .get_typed_with_source_or_fallback::<Value, _, _, E>(key, fallback)
            .await?;
        Ok((data.unwrap_or(Value::Null), source))
    }

    /// Get access to the configuration (useful for testing)
    pub fn config(&self) -> &HyperCacheConfig {
        &self.config
    }

    // ── Core cache access — Redis → S3 cascade with generic deserialization ──

    /// Like [`get_typed_with_source`], but calls `fallback` on cache miss or infrastructure error.
    /// Returns `Ok((None, _))` for the `__missing__` sentinel (fallback is not called).
    pub async fn get_typed_with_source_or_fallback<T, F, Fut, E>(
        &self,
        key: &KeyType,
        fallback: F,
    ) -> Result<(Option<T>, CacheSource), E>
    where
        T: DeserializeOwned,
        F: FnOnce() -> Fut,
        Fut: std::future::Future<Output = Result<Option<T>, E>>,
        E: From<HyperCacheError>,
    {
        match self.get_typed_with_source::<T>(key).await {
            Ok((data, source)) => Ok((data, source)),
            Err(HyperCacheError::CacheMiss) => {
                debug!("Cache miss for key {}, trying fallback", key);

                match fallback().await? {
                    Some(value) => {
                        inc(
                            HYPERCACHE_COUNTER_NAME,
                            &[
                                ("result".to_string(), "hit_fallback".to_string()),
                                ("namespace".to_string(), self.config.namespace.clone()),
                                ("value".to_string(), self.config.object_name.clone()),
                            ],
                            1,
                        );
                        Ok((Some(value), CacheSource::Fallback))
                    }
                    None => {
                        inc(
                            TOMBSTONE_COUNTER_NAME,
                            &[
                                ("namespace".to_string(), self.config.namespace.clone()),
                                (
                                    "operation".to_string(),
                                    "hypercache_fallback_miss".to_string(),
                                ),
                                ("component".to_string(), self.config.object_name.clone()),
                            ],
                            1,
                        );

                        Err(HyperCacheError::CacheMiss.into())
                    }
                }
            }
            Err(e) => {
                // Infrastructure error — try the fallback as a resilience measure.
                debug!(
                    "Cache infrastructure error for key {}: {}, trying fallback",
                    key, e
                );

                match fallback().await? {
                    Some(value) => {
                        inc(
                            HYPERCACHE_COUNTER_NAME,
                            &[
                                ("result".to_string(), "hit_fallback_infra_error".to_string()),
                                ("namespace".to_string(), self.config.namespace.clone()),
                                ("value".to_string(), self.config.object_name.clone()),
                            ],
                            1,
                        );
                        Ok((Some(value), CacheSource::Fallback))
                    }
                    None => Err(e.into()),
                }
            }
        }
    }

    /// Fetch from cache (Redis → S3), deserializing directly into `T`.
    /// Returns `Ok((None, source))` for the `__missing__` sentinel.
    pub async fn get_typed_with_source<T: DeserializeOwned>(
        &self,
        key: &KeyType,
    ) -> Result<(Option<T>, CacheSource), HyperCacheError> {
        let redis_cache_key = self.config.get_redis_cache_key(key);

        // S3 NotFound is the only authoritative miss signal; infra errors are
        // tracked separately so callers don't tombstone keys whose backing store
        // may recover.
        let mut s3_confirmed_miss = false;
        let mut infra_error: Option<HyperCacheError> = None;

        // Try Redis first
        let redis_start = Instant::now();
        let redis_result = timeout(
            self.config.redis_timeout,
            self.try_get_typed_from_redis::<T>(&redis_cache_key),
        )
        .await;
        let redis_outcome = match &redis_result {
            Ok(Ok(_)) => "hit",
            Ok(Err(HyperCacheError::CacheMiss)) => "miss",
            Ok(Err(_)) => "error",
            Err(_) => "timeout",
        };
        histogram(
            HYPERCACHE_REDIS_OP_MS,
            &[
                ("namespace".to_string(), self.config.namespace.clone()),
                ("value".to_string(), self.config.object_name.clone()),
                ("outcome".to_string(), redis_outcome.to_string()),
            ],
            redis_start.elapsed().as_secs_f64() * 1000.0,
        );
        match redis_result {
            Ok(Ok(Some(data))) => {
                debug!(
                    cache_key = %redis_cache_key,
                    namespace = %self.config.namespace,
                    "HyperCache hit: Redis"
                );
                inc(
                    HYPERCACHE_COUNTER_NAME,
                    &[
                        ("result".to_string(), "hit_redis".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                return Ok((Some(data), CacheSource::Redis));
            }
            Ok(Ok(None)) => {
                debug!(
                    cache_key = %redis_cache_key,
                    namespace = %self.config.namespace,
                    "HyperCache hit: Redis (sentinel)"
                );
                inc(
                    HYPERCACHE_COUNTER_NAME,
                    &[
                        ("result".to_string(), "hit_redis".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                return Ok((None, CacheSource::Redis));
            }
            Ok(Err(HyperCacheError::CacheMiss)) => {
                debug!(
                    cache_key = %redis_cache_key,
                    "HyperCache Redis miss, trying S3"
                );
            }
            Ok(Err(e)) => {
                debug!(
                    cache_key = %redis_cache_key,
                    error = %e,
                    "HyperCache Redis error, trying S3"
                );
                infra_error = Some(e);
            }
            Err(_) => {
                debug!(
                    cache_key = %redis_cache_key,
                    "HyperCache Redis timeout, trying S3"
                );
                inc(
                    HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME,
                    &[
                        ("reason".to_string(), "timeout".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                infra_error = Some(HyperCacheError::Timeout("redis timeout".to_string()));
            }
        }

        // Try S3 fallback
        let s3_cache_key = self.config.get_s3_cache_key(key);
        let s3_start = Instant::now();
        let s3_result = timeout(
            self.config.s3_timeout,
            self.try_get_typed_from_s3::<T>(&s3_cache_key),
        )
        .await;
        let s3_outcome = match &s3_result {
            Ok(Ok(_)) => "hit",
            Ok(Err(HyperCacheError::S3(S3Error::NotFound(_)))) => "miss",
            Ok(Err(_)) => "error",
            Err(_) => "timeout",
        };
        histogram(
            HYPERCACHE_S3_OP_MS,
            &[
                ("namespace".to_string(), self.config.namespace.clone()),
                ("value".to_string(), self.config.object_name.clone()),
                ("outcome".to_string(), s3_outcome.to_string()),
            ],
            s3_start.elapsed().as_secs_f64() * 1000.0,
        );
        match s3_result {
            Ok(Ok((data, raw_json))) => {
                debug!(
                    cache_key = %s3_cache_key,
                    namespace = %self.config.namespace,
                    "HyperCache hit: S3"
                );
                // One branch decides both outcomes, so a repair can never happen on a
                // source the caller was told to distrust. Reaching S3 on a Redis error or
                // timeout means the tier is degraded, and piling detached repair writes
                // onto it would only add load.
                let source = match infra_error {
                    None => {
                        self.spawn_read_repair(redis_cache_key, raw_json);
                        CacheSource::S3
                    }
                    Some(_) => CacheSource::S3AfterRedisError,
                };
                inc(
                    HYPERCACHE_COUNTER_NAME,
                    &[
                        ("result".to_string(), "hit_s3".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                return Ok((Some(data), source));
            }
            Ok(Err(HyperCacheError::S3(S3Error::NotFound(_)))) => {
                debug!(
                    cache_key = %s3_cache_key,
                    "HyperCache S3 confirmed miss"
                );
                s3_confirmed_miss = true;
            }
            Ok(Err(e)) => {
                debug!(
                    cache_key = %s3_cache_key,
                    error = %e,
                    "HyperCache S3 error"
                );
                if infra_error.is_none() {
                    infra_error = Some(e);
                }
            }
            Err(_) => {
                debug!(
                    cache_key = %s3_cache_key,
                    "HyperCache S3 timeout"
                );
                if infra_error.is_none() {
                    infra_error = Some(HyperCacheError::Timeout("s3 timeout".to_string()));
                }
            }
        }

        // Parse errors (Json/Pickle) are persistent, not transient: surface them
        // before checking s3_confirmed_miss so callers can tombstone the corruption
        // even when the other tier reports NotFound.
        let infra_error = match infra_error {
            Some(e @ (HyperCacheError::Json(_) | HyperCacheError::Pickle(_))) => {
                debug!(
                    redis_key = %redis_cache_key,
                    s3_key = %s3_cache_key,
                    namespace = %self.config.namespace,
                    error = %e,
                    "HyperCache parse error observed; surfacing over any S3 miss"
                );
                inc(
                    HYPERCACHE_COUNTER_NAME,
                    &[
                        ("result".to_string(), "infra_error".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                return Err(e);
            }
            other => other,
        };

        if s3_confirmed_miss {
            debug!(
                redis_key = %redis_cache_key,
                s3_key = %s3_cache_key,
                namespace = %self.config.namespace,
                "HyperCache confirmed miss: key not found in S3"
            );
            inc(
                HYPERCACHE_COUNTER_NAME,
                &[
                    ("result".to_string(), "missing".to_string()),
                    ("namespace".to_string(), self.config.namespace.clone()),
                    ("value".to_string(), self.config.object_name.clone()),
                ],
                1,
            );
            Err(HyperCacheError::CacheMiss)
        } else if let Some(e) = infra_error {
            debug!(
                redis_key = %redis_cache_key,
                s3_key = %s3_cache_key,
                namespace = %self.config.namespace,
                error = %e,
                "HyperCache infrastructure failure: both tiers had errors, not a confirmed miss"
            );
            inc(
                HYPERCACHE_COUNTER_NAME,
                &[
                    ("result".to_string(), "infra_error".to_string()),
                    ("namespace".to_string(), self.config.namespace.clone()),
                    ("value".to_string(), self.config.object_name.clone()),
                ],
                1,
            );
            Err(e)
        } else {
            // Neither s3_confirmed_miss nor infra_error was set. This
            // should not happen — emit a tombstone metric so we notice.
            // Return a Timeout rather than CacheMiss so callers don't
            // tombstone the key in a negative cache.
            inc(
                TOMBSTONE_COUNTER_NAME,
                &[
                    ("namespace".to_string(), self.config.namespace.clone()),
                    (
                        "operation".to_string(),
                        "get_with_source_unexpected_state".to_string(),
                    ),
                    ("component".to_string(), self.config.object_name.clone()),
                ],
                1,
            );
            Err(HyperCacheError::Timeout(
                "unexpected: no tier set s3_confirmed_miss or infra_error".to_string(),
            ))
        }
    }

    /// Like [`get`], but deserializes directly into `T`. Returns `None` for sentinel.
    pub async fn get_typed<T: DeserializeOwned>(
        &self,
        key: &KeyType,
    ) -> Result<Option<T>, HyperCacheError> {
        let (data, _source) = self.get_typed_with_source::<T>(key).await?;
        Ok(data)
    }

    /// Fetch from Redis only — no S3 fallback and no read repair. For callers that
    /// need to observe exactly what the Redis tier holds right now (e.g. shadow
    /// verification comparing a fresh build against the live entry) rather than the
    /// best available copy. Returns `Ok(None)` for the `__missing__` sentinel and
    /// `Err(HyperCacheError::CacheMiss)` when the key is absent; a Redis timeout
    /// surfaces as `Err(HyperCacheError::Timeout)` instead of cascading to S3.
    pub async fn get_typed_from_redis<T: DeserializeOwned>(
        &self,
        key: &KeyType,
    ) -> Result<Option<T>, HyperCacheError> {
        let redis_cache_key = self.config.get_redis_cache_key(key);
        match timeout(
            self.config.redis_timeout,
            self.try_get_typed_from_redis::<T>(&redis_cache_key),
        )
        .await
        {
            Ok(result) => result,
            Err(_) => Err(HyperCacheError::Timeout("redis timeout".to_string())),
        }
    }

    /// Read the companion ETag string for `key` from Redis, if present.
    ///
    /// The ETag is written atomically alongside the payload by `HyperCacheWriter::set_with_etag`
    /// and Django's `HyperCache._set_cache_value_redis` (when `enable_etag=True`). It serves as a
    /// cheap version tag for downstream in-memory caches that want to skip the payload fetch +
    /// deserialization on a hit.
    ///
    /// Returns `Ok(None)` when the ETag key is genuinely absent — the team uses the
    /// `__missing__` sentinel, the hypercache entry was created before `enable_etag` was on, or
    /// payload/etag TTLs drifted apart. Returns `Err` for infrastructure errors so callers can
    /// distinguish "no version available" from "couldn't reach Redis".
    pub async fn get_etag(&self, key: &KeyType) -> Result<Option<String>, HyperCacheError> {
        let etag_key = format!(
            "{}{}",
            self.config.get_redis_cache_key(key),
            ETAG_KEY_SUFFIX
        );
        match timeout(self.config.redis_timeout, self.redis_client.get(etag_key)).await {
            Ok(Ok(s)) if !s.is_empty() => Ok(Some(s)),
            Ok(Ok(_)) => Ok(None),
            Ok(Err(common_redis::CustomRedisError::NotFound)) => Ok(None),
            Ok(Err(e)) => Err(HyperCacheError::Redis(e)),
            Err(_) => Err(HyperCacheError::Timeout("etag redis timeout".to_string())),
        }
    }

    // ── Internal helpers ──

    /// Pickle-decode the Redis value and detect the `__missing__` sentinel.
    async fn try_get_json_string_from_redis(
        &self,
        cache_key: &str,
    ) -> Result<RawJsonResult, HyperCacheError> {
        // The Redis client's get_raw_bytes already handles zstd decompression,
        // so we receive Pickle(JSON) data directly.
        match self.redis_client.get_raw_bytes(cache_key.to_string()).await {
            Ok(raw_bytes) => {
                match serde_pickle::from_slice::<String>(&raw_bytes, Default::default()) {
                    Ok(json_string) => {
                        if json_string == HYPER_CACHE_EMPTY_VALUE {
                            return Ok(RawJsonResult::Empty);
                        }
                        Ok(RawJsonResult::Json(json_string))
                    }
                    Err(e) => {
                        debug!(
                            "Failed to deserialize pickle from Redis data for key '{}': {}",
                            cache_key, e
                        );
                        inc(
                            HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME,
                            &[
                                ("reason".to_string(), "pickle_error".to_string()),
                                ("namespace".to_string(), self.config.namespace.clone()),
                                ("value".to_string(), self.config.object_name.clone()),
                            ],
                            1,
                        );
                        Err(HyperCacheError::Pickle(e.to_string()))
                    }
                }
            }
            Err(common_redis::CustomRedisError::NotFound) => {
                debug!("Key not found in Redis for key '{}'", cache_key,);
                inc(
                    HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME,
                    &[
                        ("reason".to_string(), "not_found".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                Err(HyperCacheError::CacheMiss)
            }
            Err(e) => {
                // Infrastructure error — surface it so callers can distinguish
                // it from a confirmed miss.
                debug!(
                    "Failed to get raw bytes from Redis for key '{}': {}",
                    cache_key, e
                );
                inc(
                    HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME,
                    &[
                        ("reason".to_string(), "get_error".to_string()),
                        ("namespace".to_string(), self.config.namespace.clone()),
                        ("value".to_string(), self.config.object_name.clone()),
                    ],
                    1,
                );
                Err(HyperCacheError::Redis(e))
            }
        }
    }

    /// Fetch the raw JSON string from S3.
    async fn try_get_json_string_from_s3(
        &self,
        cache_key: &str,
    ) -> Result<String, HyperCacheError> {
        match self
            .s3_client
            .get_string(&self.config.s3_bucket, cache_key)
            .await
        {
            Ok(body_str) => Ok(body_str),
            Err(e) => {
                debug!("Failed to get data from S3 for key '{}': {}", cache_key, e);
                Err(HyperCacheError::S3(e))
            }
        }
    }

    async fn try_get_typed_from_redis<T: DeserializeOwned>(
        &self,
        cache_key: &str,
    ) -> Result<Option<T>, HyperCacheError> {
        match self.try_get_json_string_from_redis(cache_key).await? {
            RawJsonResult::Empty => Ok(None),
            RawJsonResult::Json(json_string) => {
                let value = serde_json::from_str::<T>(&json_string).map_err(|e| {
                    debug!(
                        "Failed to parse JSON from Redis data for key '{}': {}",
                        cache_key, e
                    );
                    inc(
                        HYPERCACHE_REDIS_MISS_REASON_COUNTER_NAME,
                        &[
                            ("reason".to_string(), "json_error".to_string()),
                            ("namespace".to_string(), self.config.namespace.clone()),
                            ("value".to_string(), self.config.object_name.clone()),
                        ],
                        1,
                    );
                    HyperCacheError::Json(e)
                })?;
                Ok(Some(value))
            }
        }
    }

    /// Returns the deserialized value alongside the raw JSON it came from, so an S3 hit can
    /// be written back into Redis verbatim without re-serializing (which would risk drifting
    /// from the bytes Django wrote).
    async fn try_get_typed_from_s3<T: DeserializeOwned>(
        &self,
        cache_key: &str,
    ) -> Result<(T, String), HyperCacheError> {
        let body_str = self.try_get_json_string_from_s3(cache_key).await?;
        let value = serde_json::from_str::<T>(&body_str).map_err(|e| {
            debug!(
                "Failed to parse JSON from S3 data for key '{}': {}",
                cache_key, e
            );
            HyperCacheError::Json(e)
        })?;
        Ok((value, body_str))
    }

    /// Best-effort write-back of an S3 hit into Redis, after Django's read path
    /// (`HyperCache.get_from_cache_with_source`, which repairs on S3 hit). Unlike Django,
    /// the caller only invokes this for a confirmed Redis miss, not for an error or
    /// timeout fall-through, where the Redis tier is degraded.
    ///
    /// Without this a key that is absent from Redis but present in S3 stays cold, so every
    /// subsequent request for it pays another S3 read until the writer next touches it. The
    /// work is detached and its failures are swallowed: a repair is an optimization, and the
    /// caller already has the value it needs.
    fn spawn_read_repair(&self, redis_cache_key: String, json_data: String) {
        let Some(ttl_seconds) = self.config.read_repair_ttl_seconds else {
            return;
        };

        let redis_client = self.redis_client.clone();
        let namespace = self.config.namespace.clone();
        let object_name = self.config.object_name.clone();

        tokio::spawn(async move {
            // NX: a repair must never overwrite an entry that already exists. The writer
            // (or a concurrent repair) may have landed a fresher value after our S3 read.
            let result = redis_client
                .set_nx_ex_with_format(
                    redis_cache_key,
                    json_data,
                    ttl_seconds,
                    writer::REDIS_FORMAT,
                )
                .await;

            let outcome = match result {
                Ok(true) => "success",
                Ok(false) => "skipped",
                Err(ref e) => {
                    debug!(
                        namespace = %namespace,
                        value = %object_name,
                        error = %e,
                        "HyperCache read repair failed"
                    );
                    "error"
                }
            };
            inc(
                HYPERCACHE_READ_REPAIR_COUNTER_NAME,
                &[
                    ("result".to_string(), outcome.to_string()),
                    ("namespace".to_string(), namespace),
                    ("value".to_string(), object_name),
                ],
                1,
            );
        });
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use common_redis::{CustomRedisError, MockRedisClient, RedisValueFormat};
    use serde_json::json;

    // Test helper functions
    fn create_test_config() -> HyperCacheConfig {
        HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        )
    }

    #[cfg(test)]
    fn create_dummy_s3_client() -> Arc<dyn S3Client + Send + Sync> {
        #[cfg(feature = "mock-client")]
        {
            // Create a dummy mock S3 client for tests that don't actually use S3
            let mut mock_s3 = MockS3Client::new();
            mock_s3.expect_get_string().returning(|_, key| {
                let key_owned = key.to_string();
                Box::pin(async move { Err(S3Error::NotFound(key_owned)) })
            });
            Arc::new(mock_s3)
        }

        #[cfg(not(feature = "mock-client"))]
        {
            // Create a real S3 client when mock-client feature is not enabled
            use aws_config::BehaviorVersion;
            use aws_sdk_s3::config::Region;

            let config = aws_sdk_s3::Config::builder()
                .behavior_version(BehaviorVersion::latest())
                .region(Region::new("us-east-1"))
                .build();
            let s3_client = AwsS3SdkClient::from_conf(config);
            Arc::new(S3Impl::new(s3_client))
        }
    }

    fn create_test_reader_with_mocks(
        mock_redis: MockRedisClient,
        mock_s3: Arc<dyn S3Client + Send + Sync>,
    ) -> HyperCacheReader {
        HyperCacheReader::new_with_s3_client(
            Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            mock_s3,
            create_test_config(),
        )
    }

    /// Mocks wired for a failed Redis read followed by an S3 hit: Redis returns
    /// `redis_err` for the fixture's key, S3 serves `s3_payload`. Tests drive whichever
    /// read API they exercise and can inspect everything the reader asked Redis to do
    /// via `redis`.
    #[cfg(feature = "mock-client")]
    struct RedisMissS3HitFixture {
        reader: HyperCacheReader,
        redis: Arc<MockRedisClient>,
        team_key: KeyType,
        cache_key: String,
    }

    #[cfg(feature = "mock-client")]
    fn redis_miss_s3_hit_fixture(
        config: HyperCacheConfig,
        s3_payload: &str,
    ) -> RedisMissS3HitFixture {
        redis_failure_s3_hit_fixture(config, s3_payload, CustomRedisError::NotFound)
    }

    #[cfg(feature = "mock-client")]
    fn redis_failure_s3_hit_fixture(
        config: HyperCacheConfig,
        s3_payload: &str,
        redis_err: CustomRedisError,
    ) -> RedisMissS3HitFixture {
        let team_key = KeyType::string("123");
        let cache_key = config.get_redis_cache_key(&team_key);
        let s3_key = config.get_s3_cache_key(&team_key);

        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_raw_bytes_ret(&cache_key, Err(redis_err));
        // Let a repair land, so the write-back path reaches its success outcome.
        mock_redis = mock_redis.set_nx_ex_ret(&cache_key, Ok(true));
        let redis = Arc::new(mock_redis);

        let mut mock_s3 = MockS3Client::new();
        mock_s3
            .expect_get_string()
            .with(predicate::eq("test-bucket"), predicate::eq(s3_key))
            .returning({
                let payload = s3_payload.to_string();
                move |_, _| {
                    let data = payload.clone();
                    Box::pin(async move { Ok(data) })
                }
            });

        let reader = HyperCacheReader::new_with_s3_client(
            redis.clone() as Arc<dyn RedisClient + Send + Sync>,
            Arc::new(mock_s3),
            config,
        );

        RedisMissS3HitFixture {
            reader,
            redis,
            team_key,
            cache_key,
        }
    }

    #[test]
    fn test_hypercache_config_new() {
        let config = HyperCacheConfig::new(
            "test_namespace".to_string(),
            "test_value".to_string(),
            "eu-west-1".to_string(),
            "custom-bucket".to_string(),
        );
        assert_eq!(config.s3_bucket, "custom-bucket");
        assert_eq!(config.s3_region, "eu-west-1");
        assert_eq!(config.s3_endpoint, None);
        assert_eq!(config.namespace, "test_namespace");
        assert_eq!(config.object_name, "test_value");
        assert_eq!(config.django_cache_version, "1"); // Default value
    }

    #[test]
    fn test_hypercache_config_with_custom_django_cache_version() {
        let config = HyperCacheConfig::with_django_cache_version(
            "test_namespace".to_string(),
            "test_value".to_string(),
            "eu-west-1".to_string(),
            "custom-bucket".to_string(),
            "3".to_string(),
        );
        assert_eq!(config.django_cache_version, "3");

        // Test that cache keys use the custom version
        let redis_key = config.get_redis_cache_key(&KeyType::string("123"));
        assert_eq!(
            redis_key,
            "posthog:3:cache/teams/123/test_namespace/test_value"
        );
    }

    #[test]
    fn test_cache_key_generation() {
        // Test ID-based cache keys
        let config = HyperCacheConfig::new(
            "flags".to_string(),
            "definitions".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        assert!(!config.token_based);

        let redis_cache_key = config.get_redis_cache_key(&KeyType::string("123"));
        assert_eq!(
            redis_cache_key,
            "posthog:1:cache/teams/123/flags/definitions"
        );

        let s3_cache_key = config.get_s3_cache_key(&KeyType::string("123"));
        assert_eq!(s3_cache_key, "cache/teams/123/flags/definitions");

        // Test token-based cache keys
        let mut token_config = config.clone();
        token_config.token_based = true;

        let redis_cache_key = token_config.get_redis_cache_key(&KeyType::string("phc_abc123"));
        assert_eq!(
            redis_cache_key,
            "posthog:1:cache/team_tokens/phc_abc123/flags/definitions"
        );

        let s3_cache_key = token_config.get_s3_cache_key(&KeyType::string("phc_abc123"));
        assert_eq!(
            s3_cache_key,
            "cache/team_tokens/phc_abc123/flags/definitions"
        );
    }

    #[test]
    fn test_keytype_int_key() {
        let config = HyperCacheConfig::new(
            "flags".to_string(),
            "definitions".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let int_key = KeyType::int(999);

        // Test Redis cache key (includes Django prefix)
        let redis_cache_key = config.get_redis_cache_key(&int_key);
        assert_eq!(
            redis_cache_key,
            "posthog:1:cache/teams/999/flags/definitions"
        );

        // Test S3 cache key (no prefix)
        let s3_cache_key = config.get_s3_cache_key(&int_key);
        assert_eq!(s3_cache_key, "cache/teams/999/flags/definitions");
    }

    #[test]
    fn test_keytype_from_conversions() {
        let str_key: KeyType = "test_string".into();
        let string_key: KeyType = "test_string".to_string().into();
        let team_id_key: KeyType = 123i32.into();
        let i32_key: KeyType = 456i32.into();

        match str_key {
            KeyType::String(s) => assert_eq!(s, "test_string"),
            _ => panic!("Expected String variant"),
        }

        match string_key {
            KeyType::String(s) => assert_eq!(s, "test_string"),
            _ => panic!("Expected String variant"),
        }

        match team_id_key {
            KeyType::Int(i) => assert_eq!(i, 123),
            _ => panic!("Expected Int variant"),
        }

        match i32_key {
            KeyType::Int(i) => assert_eq!(i, 456),
            _ => panic!("Expected Int variant"),
        }
    }

    #[tokio::test]
    async fn test_get_with_source_empty_value() {
        let team_key = KeyType::string("123");
        let expected_data = "__missing__";

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        let mut mock_redis = MockRedisClient::new();
        // Simulate Django's pickle format for empty value
        let pickled_bytes = serde_pickle::to_vec(&expected_data, Default::default()).unwrap();
        mock_redis = mock_redis.get_raw_bytes_ret(&expected_cache_key, Ok(pickled_bytes));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        let (result, source) = reader.get_with_source(&team_key).await.unwrap();
        assert_eq!(source, CacheSource::Redis);
        assert_eq!(result, Value::Null);
    }

    #[test]
    fn test_hypercache_error_conversion() {
        let cache_miss = HyperCacheError::CacheMiss;
        let redis_error = HyperCacheError::Redis(CustomRedisError::NotFound);
        let s3_error = HyperCacheError::S3(S3Error::OperationFailed("S3 error".to_string()));
        let json_error =
            HyperCacheError::Json(serde_json::from_str::<Value>("invalid").unwrap_err());
        let timeout_error = HyperCacheError::Timeout("Timeout".to_string());

        assert!(matches!(cache_miss, HyperCacheError::CacheMiss));
        assert!(matches!(redis_error, HyperCacheError::Redis(_)));
        assert!(matches!(s3_error, HyperCacheError::S3(_)));
        assert!(matches!(json_error, HyperCacheError::Json(_)));
        assert!(matches!(timeout_error, HyperCacheError::Timeout(_)));
    }

    #[tokio::test]
    async fn test_get_with_source_redis_hit() {
        let team_key = KeyType::string("123");
        let test_data = json!({"key": "value", "nested": {"data": "test"}});
        let test_data_str = serde_json::to_string(&test_data).unwrap();

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        let mut mock_redis = MockRedisClient::new();
        // Simulate Django's pickle format (uncompressed)
        let pickled_bytes = serde_pickle::to_vec(&test_data_str, Default::default()).unwrap();
        mock_redis = mock_redis.get_raw_bytes_ret(&expected_cache_key, Ok(pickled_bytes));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        let (result, source) = reader.get_with_source(&team_key).await.unwrap();
        assert_eq!(source, CacheSource::Redis);
        assert_eq!(result, test_data);
    }

    #[tokio::test]
    #[cfg(feature = "mock-client")]
    async fn test_get_with_source_redis_miss_s3_hit() {
        let test_data = json!({"key": "value", "nested": {"data": "test"}});
        let fixture = redis_miss_s3_hit_fixture(
            create_test_config(),
            &serde_json::to_string(&test_data).unwrap(),
        );

        let (result, source) = fixture
            .reader
            .get_with_source(&fixture.team_key)
            .await
            .unwrap();
        assert_eq!(source, CacheSource::S3);
        assert_eq!(result, test_data);
    }

    #[tokio::test]
    async fn test_get_with_source_complete_miss() {
        let team_key = KeyType::string("123");

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns NotFound
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::NotFound));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        // Both Redis and S3 miss should result in CacheMiss error
        let result = reader.get_with_source(&team_key).await;
        assert!(result.is_err());
        assert!(matches!(result.unwrap_err(), HyperCacheError::CacheMiss));
    }

    #[tokio::test]
    async fn test_get_with_source_or_fallback_cache_hit_redis() {
        // Test that fallback is not called when Redis cache hits
        let team_key = KeyType::string("123");
        let test_data = json!({"key": "value", "nested": {"data": "test"}});
        let test_data_str = serde_json::to_string(&test_data).unwrap();

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        let mut mock_redis = MockRedisClient::new();
        // Simulate Django's pickle format (uncompressed)
        let pickled_bytes = serde_pickle::to_vec(&test_data_str, Default::default()).unwrap();
        mock_redis = mock_redis.get_raw_bytes_ret(&expected_cache_key, Ok(pickled_bytes));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        let result: Result<(Value, CacheSource), HyperCacheError> = reader
            .get_with_source_or_fallback(&team_key, || async {
                // If this is called, the test should fail because we expect cache hit
                panic!("Fallback should not be called when cache hits!");
            })
            .await;

        assert!(result.is_ok());
        let (data, source) = result.unwrap();
        assert_eq!(source, CacheSource::Redis);
        assert_eq!(data, test_data);
    }

    #[tokio::test]
    #[cfg(feature = "mock-client")]
    async fn test_get_with_source_or_fallback_cache_hit_s3() {
        // Test that fallback is not called when S3 cache hits (after Redis miss)
        let test_data = json!({"key": "value", "nested": {"data": "test"}});
        let fixture = redis_miss_s3_hit_fixture(
            create_test_config(),
            &serde_json::to_string(&test_data).unwrap(),
        );

        let result: Result<(Value, CacheSource), HyperCacheError> = fixture
            .reader
            .get_with_source_or_fallback(&fixture.team_key, || async {
                // If this is called, the test should fail because we expect cache hit
                panic!("Fallback should not be called when S3 cache hits!");
            })
            .await;

        assert!(result.is_ok());
        let (data, source) = result.unwrap();
        assert_eq!(source, CacheSource::S3);
        assert_eq!(data, test_data);
    }

    /// Drive one fixture read to its S3 hit and return every call the reader made to Redis.
    /// Callers run under paused tokio time: the sleep parks this task, the runtime drives the
    /// detached repair task to idle, and only then advances the clock, so any spawned repair
    /// has landed before inspection.
    #[cfg(feature = "mock-client")]
    async fn redis_calls_after_s3_hit(
        fixture: &RedisMissS3HitFixture,
        expected_source: CacheSource,
    ) -> Vec<common_redis::MockRedisCall> {
        let (_, source) = fixture
            .reader
            .get_with_source(&fixture.team_key)
            .await
            .unwrap();
        assert_eq!(source, expected_source);
        assert_eq!(source.as_log_str(), "s3");

        tokio::time::sleep(Duration::from_millis(1)).await;
        fixture.redis.get_calls()
    }

    #[tokio::test(start_paused = true)]
    #[cfg(feature = "mock-client")]
    async fn test_s3_hit_repairs_redis_when_ttl_configured() {
        let payload = r#"{"key":"value"}"#;
        let mut config = create_test_config();
        config.read_repair_ttl_seconds = Some(600);

        let fixture = redis_miss_s3_hit_fixture(config, payload);
        let calls = redis_calls_after_s3_hit(&fixture, CacheSource::S3).await;

        let repair = calls
            .iter()
            .find(|c| c.op == "set_nx_ex_with_format")
            .expect("expected the S3 hit to be written back to Redis");
        assert_eq!(repair.key, fixture.cache_key);
        match &repair.value {
            common_redis::MockRedisValue::StringWithTTLAndFormat(value, ttl, format) => {
                // Written back verbatim, in the pickle format Django reads.
                assert_eq!(value, payload);
                assert_eq!(*ttl, 600);
                assert_eq!(*format, RedisValueFormat::Pickle);
            }
            other => panic!("expected StringWithTTLAndFormat, got {other:?}"),
        }
    }

    #[tokio::test(start_paused = true)]
    #[cfg(feature = "mock-client")]
    async fn test_s3_hit_does_not_repair_redis_by_default() {
        // The reader is read-only unless a caller opts in.
        let fixture = redis_miss_s3_hit_fixture(create_test_config(), r#"{"key":"value"}"#);
        let calls = redis_calls_after_s3_hit(&fixture, CacheSource::S3).await;
        assert!(calls.iter().all(|c| c.op != "set_nx_ex_with_format"));
    }

    #[tokio::test(start_paused = true)]
    #[cfg(feature = "mock-client")]
    async fn test_s3_hit_does_not_repair_etag_enabled_namespace() {
        // Repairing the payload alone would leave it inconsistent with its companion etag.
        let mut config = create_test_config();
        config.read_repair_ttl_seconds = Some(600);
        config.enable_etag = true;

        let fixture = redis_miss_s3_hit_fixture(config, r#"{"key":"value"}"#);
        let calls = redis_calls_after_s3_hit(&fixture, CacheSource::S3).await;
        assert!(calls.iter().all(|c| c.op != "set_nx_ex_with_format"));
    }

    #[cfg(feature = "mock-client")]
    #[test]
    fn test_read_repair_ttl_is_capped_at_construction() {
        // A misconfigured env var (e.g. seconds vs. milliseconds) must not silently disable
        // the short-TTL invariant that bounds a resurrected orphan's lifetime.
        let mut config = create_test_config();
        config.read_repair_ttl_seconds = Some(HyperCacheReader::MAX_READ_REPAIR_TTL_SECONDS + 1);

        let reader = HyperCacheReader::new_with_s3_client(
            Arc::new(MockRedisClient::new()) as Arc<dyn RedisClient + Send + Sync>,
            create_dummy_s3_client(),
            config,
        );

        assert_eq!(
            reader.config().read_repair_ttl_seconds,
            Some(HyperCacheReader::MAX_READ_REPAIR_TTL_SECONDS)
        );
    }

    #[cfg(feature = "mock-client")]
    #[test]
    fn test_read_repair_ttl_at_max_is_unchanged() {
        let mut config = create_test_config();
        config.read_repair_ttl_seconds = Some(HyperCacheReader::MAX_READ_REPAIR_TTL_SECONDS);

        let reader = HyperCacheReader::new_with_s3_client(
            Arc::new(MockRedisClient::new()) as Arc<dyn RedisClient + Send + Sync>,
            create_dummy_s3_client(),
            config,
        );

        assert_eq!(
            reader.config().read_repair_ttl_seconds,
            Some(HyperCacheReader::MAX_READ_REPAIR_TTL_SECONDS)
        );
    }

    #[tokio::test(start_paused = true)]
    #[cfg(feature = "mock-client")]
    async fn test_s3_hit_does_not_repair_after_redis_error() {
        // Reaching S3 on a Redis error means the tier is degraded, not that the key is
        // cold; writing to a degraded tier would only add load.
        let mut config = create_test_config();
        config.read_repair_ttl_seconds = Some(600);

        let fixture =
            redis_failure_s3_hit_fixture(config, r#"{"key":"value"}"#, CustomRedisError::Timeout);
        let calls = redis_calls_after_s3_hit(&fixture, CacheSource::S3AfterRedisError).await;
        assert!(calls.iter().all(|c| c.op != "set_nx_ex_with_format"));
    }

    #[tokio::test]
    async fn test_get_with_source_or_fallback_uses_fallback() {
        // Test that fallback is called when both cache tiers miss
        let team_key = KeyType::string("123");
        let fallback_data = json!({"fallback": "data", "from": "database"});

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns NotFound
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::NotFound));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(), // S3 will also return NotFound
            config,
        };

        let fallback_data_clone = fallback_data.clone();
        let result = reader
            .get_with_source_or_fallback(&team_key, || {
                let data = fallback_data_clone.clone();
                async move { Ok::<Option<Value>, HyperCacheError>(Some(data)) }
            })
            .await;

        assert!(result.is_ok());
        let (data, source) = result.unwrap();
        assert_eq!(source, CacheSource::Fallback);
        assert_eq!(data, fallback_data);
    }

    #[tokio::test]
    async fn test_get_with_source_or_fallback_fallback_returns_none() {
        // Test when fallback returns None (data doesn't exist in database either)
        let team_key = KeyType::string("123");

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns NotFound
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::NotFound));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        let result = reader
            .get_with_source_or_fallback(&team_key, || async {
                Ok::<Option<Value>, HyperCacheError>(None)
            })
            .await;

        assert!(result.is_err());
        assert!(matches!(result.unwrap_err(), HyperCacheError::CacheMiss));
    }

    #[tokio::test]
    async fn test_get_with_source_or_fallback_fallback_error() {
        // Test when fallback returns an error
        let team_key = KeyType::string("123");

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns NotFound
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::NotFound));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        #[derive(Debug)]
        struct CustomError(String);
        impl From<HyperCacheError> for CustomError {
            fn from(e: HyperCacheError) -> Self {
                CustomError(format!("Cache error: {e}"))
            }
        }

        let result = reader
            .get_with_source_or_fallback(&team_key, || async {
                Err::<Option<Value>, CustomError>(CustomError(
                    "Database connection failed".to_string(),
                ))
            })
            .await;

        assert!(result.is_err());
        match result.unwrap_err() {
            CustomError(msg) => assert!(msg.contains("Database connection failed")),
        }
    }

    #[tokio::test]
    async fn test_get_with_source_or_fallback_no_cache_write() {
        // Verify that fallback data is NOT written to cache
        let team_key = KeyType::string("123");
        let fallback_data = json!({"fallback": "data"});

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns NotFound initially
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::NotFound));

        // Important: mock_redis should NOT receive any set/setex calls
        // If it did, the test would panic due to unexpected method calls

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_dummy_s3_client(),
            config,
        };

        let fallback_data_clone = fallback_data.clone();
        let result = reader
            .get_with_source_or_fallback(&team_key, || {
                let data = fallback_data_clone.clone();
                async move { Ok::<Option<Value>, HyperCacheError>(Some(data)) }
            })
            .await;

        assert!(result.is_ok());
        let (data, source) = result.unwrap();
        assert_eq!(source, CacheSource::Fallback);
        assert_eq!(data, fallback_data);

        // If we got here without panicking, it means no cache write was attempted
        // (MockRedisClient would panic on unexpected method calls)
    }

    /// Helper: create an S3 mock that returns OperationFailed for all gets.
    /// This simulates an S3 outage (as opposed to the dummy client which returns NotFound).
    #[cfg(feature = "mock-client")]
    fn create_failing_s3_client() -> Arc<dyn S3Client + Send + Sync> {
        let mut mock_s3 = MockS3Client::new();
        mock_s3.expect_get_string().returning(|_, _| {
            Box::pin(
                async move { Err(S3Error::OperationFailed("simulated S3 outage".to_string())) },
            )
        });
        Arc::new(mock_s3)
    }

    #[tokio::test]
    #[cfg(feature = "mock-client")]
    async fn test_get_with_source_or_fallback_infra_error_calls_fallback() {
        // When both Redis and S3 have infrastructure errors (not confirmed
        // misses), get_with_source returns the infra error. The fallback
        // arm in get_with_source_or_fallback should still call the fallback
        // so callers like feature-flags can fall back to Postgres.
        let team_key = KeyType::string("123");
        let fallback_data = json!({"team": "from_postgres"});

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        // Redis returns a Timeout (infrastructure error, not NotFound)
        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::Timeout));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_failing_s3_client(),
            config,
        };

        let fallback_data_clone = fallback_data.clone();
        let result = reader
            .get_with_source_or_fallback(&team_key, || {
                let data = fallback_data_clone.clone();
                async move { Ok::<Option<Value>, HyperCacheError>(Some(data)) }
            })
            .await;

        assert!(result.is_ok());
        let (data, source) = result.unwrap();
        assert_eq!(source, CacheSource::Fallback);
        assert_eq!(data, fallback_data);
    }

    #[tokio::test]
    #[cfg(feature = "mock-client")]
    async fn test_get_with_source_or_fallback_infra_error_fallback_none_returns_infra_error() {
        // When both cache tiers have infrastructure errors and the fallback
        // returns None, the original infrastructure error should be surfaced
        // (not CacheMiss).
        let team_key = KeyType::string("123");

        let config = HyperCacheConfig::new(
            "test".to_string(),
            "test".to_string(),
            "us-east-1".to_string(),
            "test-bucket".to_string(),
        );
        let expected_cache_key = config.get_redis_cache_key(&team_key);

        let mut mock_redis = MockRedisClient::new();
        mock_redis = mock_redis.get_ret(&expected_cache_key, Err(CustomRedisError::Timeout));

        let reader = HyperCacheReader {
            redis_client: Arc::new(mock_redis) as Arc<dyn RedisClient + Send + Sync>,
            s3_client: create_failing_s3_client(),
            config,
        };

        let result = reader
            .get_with_source_or_fallback(&team_key, || async {
                Ok::<Option<Value>, HyperCacheError>(None)
            })
            .await;

        assert!(result.is_err());
        // Should be the Redis infra error, not CacheMiss
        assert!(
            matches!(
                result.unwrap_err(),
                HyperCacheError::Redis(CustomRedisError::Timeout)
            ),
            "should surface the original infrastructure error, not CacheMiss"
        );
    }

    // ── Tests for typed deserialization methods ──

    #[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
    struct TestFlags {
        flags: Vec<TestFlag>,
        team_id: i32,
    }

    #[derive(Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
    struct TestFlag {
        id: i32,
        key: String,
        active: bool,
    }

    fn make_test_flags() -> TestFlags {
        TestFlags {
            flags: vec![
                TestFlag {
                    id: 1,
                    key: "beta-feature".to_string(),
                    active: true,
                },
                TestFlag {
                    id: 2,
                    key: "dark-mode".to_string(),
                    active: false,
                },
            ],
            team_id: 42,
        }
    }

    fn pickle_json(value: &impl serde::Serialize) -> Vec<u8> {
        let json_string = serde_json::to_string(value).unwrap();
        serde_pickle::to_vec(&json_string, Default::default()).unwrap()
    }

    #[tokio::test]
    async fn test_get_typed_from_redis_hit() {
        let expected = make_test_flags();
        let pickled = pickle_json(&expected);

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let data = reader
            .get_typed_from_redis::<TestFlags>(&KeyType::int(42))
            .await
            .unwrap();

        assert_eq!(data, Some(expected));
    }

    #[tokio::test]
    async fn test_get_typed_from_redis_sentinel_returns_none() {
        let sentinel_pickled =
            serde_pickle::to_vec(&HYPER_CACHE_EMPTY_VALUE, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(99));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(sentinel_pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let data = reader
            .get_typed_from_redis::<TestFlags>(&KeyType::int(99))
            .await
            .unwrap();

        assert_eq!(data, None);
    }

    /// A Redis miss must surface as `CacheMiss` even when S3 holds the value —
    /// the whole point of the Redis-only getter is that it never cascades.
    #[cfg(feature = "mock-client")]
    #[tokio::test]
    async fn test_get_typed_from_redis_miss_does_not_fall_back_to_s3() {
        let payload = serde_json::to_string(&make_test_flags()).unwrap();
        let fixture = redis_miss_s3_hit_fixture(create_test_config(), &payload);

        // First prove S3 really holds the value — the cascading read returns it —
        // so the Redis-only miss below is a refusal to fall back, not a vacuous
        // pass against an empty fixture.
        let (cascaded, source) = fixture
            .reader
            .get_typed_with_source::<TestFlags>(&fixture.team_key)
            .await
            .unwrap();
        assert_eq!(source, CacheSource::S3);
        assert_eq!(cascaded, Some(make_test_flags()));

        let result = fixture
            .reader
            .get_typed_from_redis::<TestFlags>(&fixture.team_key)
            .await;

        assert!(matches!(result, Err(HyperCacheError::CacheMiss)));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_redis_hit() {
        let expected = make_test_flags();
        let pickled = pickle_json(&expected);

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let (data, source) = reader
            .get_typed_with_source::<TestFlags>(&KeyType::int(42))
            .await
            .unwrap();

        assert_eq!(source, CacheSource::Redis);
        assert_eq!(data, Some(expected));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_sentinel_returns_none() {
        let sentinel_pickled =
            serde_pickle::to_vec(&HYPER_CACHE_EMPTY_VALUE, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(99));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(sentinel_pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let (data, source) = reader
            .get_typed_with_source::<TestFlags>(&KeyType::int(99))
            .await
            .unwrap();

        assert_eq!(source, CacheSource::Redis);
        assert!(data.is_none(), "sentinel should return None");
    }

    #[cfg(feature = "mock-client")]
    #[tokio::test]
    async fn test_get_typed_with_source_redis_miss_s3_hit() {
        let expected = make_test_flags();
        let fixture = redis_miss_s3_hit_fixture(
            create_test_config(),
            &serde_json::to_string(&expected).unwrap(),
        );

        let (data, source) = fixture
            .reader
            .get_typed_with_source::<TestFlags>(&fixture.team_key)
            .await
            .unwrap();

        assert_eq!(source, CacheSource::S3);
        assert_eq!(data, Some(expected));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_redis_json_error_surfaces_over_s3_miss() {
        let invalid_json = "not valid json {{{";
        let pickled = serde_pickle::to_vec(&invalid_json, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let result = reader
            .get_typed_with_source::<TestFlags>(&KeyType::int(42))
            .await;

        assert!(
            matches!(result.unwrap_err(), HyperCacheError::Json(_)),
            "parse error must surface over S3 NotFound"
        );
    }

    #[cfg(feature = "mock-client")]
    #[tokio::test]
    async fn test_get_typed_with_source_redis_json_error_falls_through_to_s3_hit() {
        let expected = make_test_flags();
        let json_string = serde_json::to_string(&expected).unwrap();

        let invalid_json = "not valid json {{{";
        let pickled = serde_pickle::to_vec(&invalid_json, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));
        let redis_handle = mock_redis.clone();

        let mut mock_s3 = MockS3Client::new();
        let s3_key = create_test_config().get_s3_cache_key(&KeyType::int(42));
        let json_clone = json_string.clone();
        mock_s3
            .expect_get_string()
            .with(
                predicate::eq("test-bucket".to_string()),
                predicate::eq(s3_key),
            )
            .returning(move |_, _| {
                let val = json_clone.clone();
                Box::pin(async move { Ok(val) })
            });

        let reader = create_test_reader_with_mocks(mock_redis, Arc::new(mock_s3));
        let (data, source) = reader
            .get_typed_with_source::<TestFlags>(&KeyType::int(42))
            .await
            .unwrap();

        assert_eq!(source, CacheSource::S3AfterRedisError);
        assert_eq!(data, Some(expected));
        assert!(
            redis_handle.get_calls().iter().any(|c| c.key == cache_key),
            "Redis must be consulted before S3"
        );
    }

    #[cfg(feature = "mock-client")]
    #[tokio::test]
    async fn test_get_typed_with_source_redis_pickle_error_falls_through_to_s3_hit() {
        let expected = make_test_flags();
        let json_string = serde_json::to_string(&expected).unwrap();

        // Raw non-pickle bytes make serde_pickle::from_slice::<String> fail.
        let non_pickle_bytes = b"this is not pickle data".to_vec();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(non_pickle_bytes));
        let redis_handle = mock_redis.clone();

        let mut mock_s3 = MockS3Client::new();
        let s3_key = create_test_config().get_s3_cache_key(&KeyType::int(42));
        let json_clone = json_string.clone();
        mock_s3
            .expect_get_string()
            .with(
                predicate::eq("test-bucket".to_string()),
                predicate::eq(s3_key),
            )
            .returning(move |_, _| {
                let val = json_clone.clone();
                Box::pin(async move { Ok(val) })
            });

        let reader = create_test_reader_with_mocks(mock_redis, Arc::new(mock_s3));
        let (data, source) = reader
            .get_typed_with_source::<TestFlags>(&KeyType::int(42))
            .await
            .unwrap();

        assert_eq!(source, CacheSource::S3AfterRedisError);
        assert_eq!(data, Some(expected));
        assert!(
            redis_handle.get_calls().iter().any(|c| c.key == cache_key),
            "Redis must be consulted before S3"
        );
    }

    #[tokio::test]
    async fn test_get_typed_from_redis_json_parse_error() {
        // Test the internal helper directly to verify JSON errors surface correctly
        let invalid_json = "not valid json {{{";
        let pickled = serde_pickle::to_vec(&invalid_json, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let result = reader
            .try_get_typed_from_redis::<TestFlags>(&cache_key)
            .await;

        assert!(result.is_err());
        assert!(matches!(result.unwrap_err(), HyperCacheError::Json(_)));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_or_fallback_cache_hit() {
        let expected = make_test_flags();
        let pickled = pickle_json(&expected);

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let (data, source) = reader
            .get_typed_with_source_or_fallback::<TestFlags, _, _, HyperCacheError>(
                &KeyType::int(42),
                || async { panic!("fallback should not be called") },
            )
            .await
            .unwrap();

        assert_eq!(source, CacheSource::Redis);
        assert_eq!(data, Some(expected));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_or_fallback_uses_fallback() {
        let expected = make_test_flags();
        let expected_clone = expected.clone();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Err(CustomRedisError::NotFound));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let (data, source) = reader
            .get_typed_with_source_or_fallback::<TestFlags, _, _, HyperCacheError>(
                &KeyType::int(42),
                || async move { Ok(Some(expected_clone)) },
            )
            .await
            .unwrap();

        assert_eq!(source, CacheSource::Fallback);
        assert_eq!(data, Some(expected));
    }

    #[tokio::test]
    async fn test_get_typed_with_source_or_fallback_sentinel_no_fallback() {
        let sentinel_pickled =
            serde_pickle::to_vec(&HYPER_CACHE_EMPTY_VALUE, Default::default()).unwrap();

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(99));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(sentinel_pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let (data, source) = reader
            .get_typed_with_source_or_fallback::<TestFlags, _, _, HyperCacheError>(
                &KeyType::int(99),
                || async { panic!("fallback should not be called for sentinel") },
            )
            .await
            .unwrap();

        assert_eq!(source, CacheSource::Redis);
        assert!(data.is_none());
    }

    #[tokio::test]
    async fn test_get_typed_simple() {
        let expected = make_test_flags();
        let pickled = pickle_json(&expected);

        let mut mock_redis = MockRedisClient::new();
        let cache_key = create_test_config().get_redis_cache_key(&KeyType::int(42));
        mock_redis.get_raw_bytes_ret(&cache_key, Ok(pickled));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let data = reader
            .get_typed::<TestFlags>(&KeyType::int(42))
            .await
            .unwrap();

        assert_eq!(data, Some(expected));
    }

    /// Verifies the ETag round-trips through `get_etag` against the same suffix
    /// `HyperCacheWriter::set_with_etag` writes to. Pinning the suffix here
    /// guards against a future split between the writer and reader: if either
    /// side drifts off `:etag`, `FlagDefinitionsCache` would silently turn into
    /// a perma-miss cache, which is the failure mode this test exists to catch.
    #[tokio::test]
    async fn test_get_etag_returns_value_when_present() {
        let mut mock_redis = MockRedisClient::new();
        let etag_key = format!(
            "{}{}",
            create_test_config().get_redis_cache_key(&KeyType::int(42)),
            ETAG_KEY_SUFFIX
        );
        mock_redis.get_ret(&etag_key, Ok("0123456789abcdef".to_string()));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let etag = reader.get_etag(&KeyType::int(42)).await.unwrap();

        assert_eq!(etag.as_deref(), Some("0123456789abcdef"));
    }

    /// `__missing__` writes call `delete_etag`, so the etag key is absent for
    /// teams with no flags. Returning `Ok(None)` lets `FlagDefinitionsCache`
    /// take the no-cache path without surfacing this as an error.
    #[tokio::test]
    async fn test_get_etag_returns_none_when_key_absent() {
        let mock_redis = MockRedisClient::new();
        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());

        let etag = reader.get_etag(&KeyType::int(42)).await.unwrap();
        assert!(etag.is_none());
    }

    /// Infrastructure errors (timeout, connection refused) must propagate so the
    /// caller can decide whether to fall through to the payload path or surface
    /// the error. Returning `Ok(None)` here would silently degrade the version-
    /// key fast path without any signal in metrics.
    #[tokio::test]
    async fn test_get_etag_propagates_redis_error() {
        let mut mock_redis = MockRedisClient::new();
        let etag_key = format!(
            "{}{}",
            create_test_config().get_redis_cache_key(&KeyType::int(42)),
            ETAG_KEY_SUFFIX
        );
        mock_redis.get_ret(&etag_key, Err(common_redis::CustomRedisError::Timeout));

        let reader = create_test_reader_with_mocks(mock_redis, create_dummy_s3_client());
        let result = reader.get_etag(&KeyType::int(42)).await;

        assert!(matches!(result, Err(HyperCacheError::Redis(_))));
    }
}