:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use crate::{
api::errors::FlagError,
flags::{
feature_flag_list::LenientHypercacheFlagsWrapper,
flag_definitions_cache::FlagDefinitionsCache,
flag_models::{HypercacheFlagsWrapper, PreparedFlagDefinitions},
},
handler::canonical_log::with_canonical_log,
metrics::consts::{
DB_TEAM_READS_COUNTER, PG_TEAM_FALLBACK_SKIPPED_COUNTER, TEAM_CACHE_HIT_COUNTER,
TEAM_NEGATIVE_CACHE_HIT_COUNTER, TOKEN_VALIDATION_ERRORS_COUNTER, TOMBSTONE_COUNTER,
},
team::team_models::Team,
};
use common_cache::NegativeCache;
use common_database::PostgresReader;
use common_hypercache::{
CacheSource, HyperCacheError, HyperCacheReader, KeyType, HYPERCACHE_COUNTER_NAME,
};
use common_metrics::inc;
use common_redis::Client as RedisClient;
use common_types::TeamId;
use metrics::counter;
use std::sync::Arc;
/// Result of fetching feature flags, including cache source information.
#[derive(Debug, Clone)]
pub struct FlagResult {
/// Pre-compiled flag definitions (deserialized + regex-compiled), shared across requests.
pub prepared: Arc<PreparedFlagDefinitions>,
/// The source of the flags data (Redis, S3, or Fallback/PostgreSQL).
pub cache_source: common_hypercache::CacheSource,
}
/// Service layer for handling feature flag operations
pub struct FlagService {
/// Shared Redis client for non-critical path operations
/// Reserved for future use (analytics counters, billing limits, etc.)
#[allow(dead_code)]
shared_redis_client: Arc<dyn RedisClient + Send + Sync>,
pg_client: PostgresReader,
/// HyperCache reader for fetching team metadata from Redis/S3
team_hypercache_reader: Arc<HyperCacheReader>,
/// HyperCache reader for fetching flags from Redis/S3
/// Arc-wrapped to allow sharing across requests
flags_hypercache_reader: Arc<HyperCacheReader>,
/// In-memory cache for deserialized + regex-compiled flag definitions
flag_definitions_cache: Arc<FlagDefinitionsCache>,
/// In-memory negative cache for invalid API tokens
team_negative_cache: NegativeCache,
/// When true, skip PG fallback for team token lookups
skip_pg_team_fallback: bool,
}
impl FlagService {
pub fn new(
shared_redis_client: Arc<dyn RedisClient + Send + Sync>,
pg_client: PostgresReader,
team_hypercache_reader: Arc<HyperCacheReader>,
flags_hypercache_reader: Arc<HyperCacheReader>,
flag_definitions_cache: Arc<FlagDefinitionsCache>,
team_negative_cache: NegativeCache,
skip_pg_team_fallback: bool,
) -> Self {
Self {
shared_redis_client,
pg_client,
team_hypercache_reader,
flags_hypercache_reader,
flag_definitions_cache,
team_negative_cache,
skip_pg_team_fallback,
}
}
/// Deprecated: use `verify_token_and_get_team` instead, which returns the Team
/// directly and avoids a redundant cache lookup.
#[allow(dead_code)]
pub async fn verify_token(&self, token: &str) -> Result<String, FlagError> {
self.verify_token_and_get_team(token)
.await
.map(|_| token.to_string())
}
/// Verifies the Project API token and returns the Team.
///
/// This combines token verification with team fetching to avoid a redundant
/// cache lookup — callers get the Team directly instead of re-fetching it.
/// Invalid tokens are tracked in a negative cache to avoid repeated lookups
/// against Redis/S3/PG for tokens that don't correspond to any team.
pub async fn verify_token_and_get_team(&self, token: &str) -> Result<Team, FlagError> {
if self.team_negative_cache.contains(token) {
with_canonical_log(|log| log.team_cache_source = Some("negative_cache"));
inc(TEAM_NEGATIVE_CACHE_HIT_COUNTER, &[], 1);
inc(
TOKEN_VALIDATION_ERRORS_COUNTER,
&[("reason".to_string(), "token_not_found".to_string())],
1,
);
return Err(FlagError::TokenValidationError);
}
match self.get_team_from_cache_or_pg(token).await {
Ok(team) => Ok(team),
Err(e) => {
tracing::warn!("Token validation failed for token '{}': {:?}", token, e);
if e.is_token_not_found() {
self.team_negative_cache.insert(token.to_string());
inc(
TOKEN_VALIDATION_ERRORS_COUNTER,
&[("reason".to_string(), "token_not_found".to_string())],
1,
);
}
Err(FlagError::TokenValidationError)
}
}
}
/// Fetches a team by its database ID.
///
/// Used when the team_id is known from authentication (e.g. a phs_ token)
/// but the team object is needed for cache lookups and response building.
pub async fn get_team_by_id(&self, team_id: i32) -> Result<Team, FlagError> {
with_canonical_log(|log| log.team_cache_source = Some("pg_by_id"));
inc(
DB_TEAM_READS_COUNTER,
&[("path".to_string(), "by_id".to_string())],
1,
);
Team::from_pg_by_id(self.pg_client.clone(), team_id)
.await
.map_err(|e| match e {
FlagError::RowNotFound => FlagError::SecretApiTokenInvalid,
other => other,
})
}
/// Fetches the team from HyperCache or the database.
///
/// Uses team_metadata HyperCache (Redis → S3 → PostgreSQL fallback).
/// PostgreSQL fallback can be disabled via `skip_pg_team_fallback`.
/// This is a read-only cache - Django handles cache writes.
pub async fn get_team_from_cache_or_pg(&self, token: &str) -> Result<Team, FlagError> {
let key = KeyType::string(token);
let skip_pg = self.skip_pg_team_fallback;
let pg_client = self.pg_client.clone();
let token_owned = token.to_string();
let (data, source) = self
.team_hypercache_reader
.get_typed_with_source_or_fallback::<Team, _, _, FlagError>(&key, || async move {
// This closure runs on cache miss (key not found in Redis and
// S3) or infrastructure errors (timeouts, connection failures)
// as a resilience fallback.
if skip_pg {
inc(PG_TEAM_FALLBACK_SKIPPED_COUNTER, &[], 1);
return Err(FlagError::TokenValidationError);
}
let team = Team::from_pg(pg_client, &token_owned).await?;
inc(DB_TEAM_READS_COUNTER, &[], 1);
Ok::<Option<Team>, FlagError>(Some(team))
})
.await?;
let team = data.ok_or(FlagError::TokenValidationError)?;
let cache_hit = !matches!(source, CacheSource::Fallback);
with_canonical_log(|log| log.team_cache_source = Some(source.as_log_str()));
inc(
TEAM_CACHE_HIT_COUNTER,
&[("cache_hit".to_string(), cache_hit.to_string())],
1,
);
Ok(team)
}
/// Fetches flags from the hypercache (Redis → S3), falling back to PostgreSQL
/// on cache miss or infra errors. A payload-level parse error (`Json`/`Pickle`)
/// hard-fails with a tombstone rather than silently serving PG data. A single
/// flag that does not parse is skipped, not fatal.
///
/// On the hot path the in-memory `FlagDefinitionsCache` is keyed on the etag
/// Django writes alongside the payload (`enable_etag=True`), so an in-memory
/// hit short-circuits the payload fetch / pickle / JSON / validation work
/// entirely. The loader passed to `get_or_load` only runs on a true miss
/// or when no etag is available.
pub async fn get_flags_from_cache_or_pg(
&self,
team_id: TeamId,
) -> Result<FlagResult, FlagError> {
let key = KeyType::int(team_id);
// Cheap version probe: a single Redis GET on a 16-byte string. On a
// Redis error we bypass `get_or_load` entirely so the request
// increments only `etag_redis_error`, not also `etag_missing` /
// `sentinel` (each request must increment exactly one `reason`).
match self.flags_hypercache_reader.get_etag(&key).await {
Ok(etag) => {
let (prepared, cache_source) = self
.flag_definitions_cache
.get_or_load(team_id, etag, || async {
self.fetch_wrapper_or_pg(team_id).await
})
.await?;
Ok(FlagResult {
prepared,
cache_source,
})
}
Err(e) => {
tracing::debug!(
team_id,
error = %e,
"etag fetch failed; bypassing in-memory cache for this request"
);
inc(
crate::metrics::consts::FLAG_DEFINITIONS_INMEM_CACHE_NO_VERSION_COUNTER,
&[("reason".to_string(), "etag_redis_error".to_string())],
1,
);
let (wrapper, cache_source) = self.fetch_wrapper_or_pg(team_id).await?;
let prepared =
crate::flags::flag_definitions_cache::compile_from_wrapper(team_id, wrapper)?;
Ok(FlagResult {
prepared,
cache_source,
})
}
}
}
/// Hypercache-then-PG payload fetch, extracted so the in-memory cache can
/// invoke it lazily inside `get_or_load`. Returns `None` for the
/// `__missing__` sentinel (team has no flags), `Some(wrapper)` otherwise.
/// A payload-level `Json`/`Pickle` parse error hard-fails with a tombstone,
/// because we never want to silently degrade to PG on data corruption. A flag
/// whose own JSON does not parse is skipped and counted instead, so one bad
/// flag does not fail the evaluation of the team's other flags.
async fn fetch_wrapper_or_pg(
&self,
team_id: TeamId,
) -> Result<(Option<HypercacheFlagsWrapper>, CacheSource), FlagError> {
let key = KeyType::int(team_id);
match self
.flags_hypercache_reader
.get_typed_with_source::<LenientHypercacheFlagsWrapper>(&key)
.await
{
Ok((data, source)) => Ok((data.map(|w| w.into_wrapper(team_id)), source)),
Err(e @ (HyperCacheError::Json(_) | HyperCacheError::Pickle(_))) => {
counter!(
TOMBSTONE_COUNTER,
"namespace" => "feature_flags",
"operation" => "hypercache_parse_error",
"component" => "flag_service",
)
.increment(1);
Err(FlagError::flag_data_parsing(format!(
"Failed to parse feature flags for team {team_id}: {e}"
)))
}
Err(e) => {
// Mirror the hit_fallback counters that `get_typed_with_source_or_fallback`
// would emit, so flags and team paths stay on identical instrumentation.
let result_label = if matches!(e, HyperCacheError::CacheMiss) {
"hit_fallback"
} else {
"hit_fallback_infra_error"
};
let hc_config = self.flags_hypercache_reader.config();
inc(
HYPERCACHE_COUNTER_NAME,
&[
("result".to_string(), result_label.to_string()),
("namespace".to_string(), hc_config.namespace.clone()),
("value".to_string(), hc_config.object_name.clone()),
],
1,
);
let wrapper = crate::flags::cache_builder::build_fallback_wrapper(
self.pg_client.clone(),
team_id,
)
.await?;
Ok((Some(wrapper), CacheSource::Fallback))
}
}
}
}
#[cfg(test)]
mod tests {
use common_cache::NegativeCache;
use rstest::rstest;
use serde_json::json;
use crate::{
flags::{
feature_flag_list::PreparedFlags,
flag_definitions_cache::FlagDefinitionsCache,
flag_models::{
EvaluationMetadata, FeatureFlag, FeatureFlagList, FeatureFlagRow, FlagFilters,
FlagPropertyGroup, HypercacheFlagsWrapper,
},
test_helpers::{hypercache_test_key, update_flags_in_hypercache},
},
properties::property_models::{OperatorType, PropertyFilter, PropertyType},
utils::test_utils::{
insert_new_team_in_redis, setup_hypercache_reader,
setup_hypercache_reader_with_mock_redis, setup_pg_reader_client, setup_redis_client,
setup_team_hypercache_reader, setup_team_hypercache_reader_with_mock_redis,
TestContext,
},
};
use super::*;
#[tokio::test]
async fn test_verify_token_and_get_team() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert new team in Redis");
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
// Test valid token returns the team
let result = flag_service
.verify_token_and_get_team(&team.api_token)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap().api_token, team.api_token);
// Test invalid token
let result = flag_service
.verify_token_and_get_team("invalid_token")
.await;
assert!(matches!(result, Err(FlagError::TokenValidationError)));
}
#[tokio::test]
async fn test_get_team_from_cache_or_pg() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert new team in Redis");
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
// Test fetching from HyperCache
let result = flag_service
.get_team_from_cache_or_pg(&team.api_token)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap().id, team.id);
}
#[tokio::test]
async fn test_get_team_from_pg_fallback() {
let redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
// Insert a team in PG but not in HyperCache
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team in pg");
let flag_service = FlagService::new(
redis_client.clone(),
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
// Test fetching from PostgreSQL (cache miss)
let result = flag_service
.get_team_from_cache_or_pg(&team.api_token)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap().id, team.id);
}
#[tokio::test]
async fn test_get_flags_from_hypercache() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert new team in Redis");
// Insert some mock flags into hypercache (new format)
let flags_vec = vec![
FeatureFlag {
id: 1,
team_id: team.id,
name: Some("Beta Feature".to_string()),
key: "beta_feature".to_string(),
has_experiment: false,
filters: FlagFilters {
non_v1: None,
groups: vec![FlagPropertyGroup {
properties: Some(vec![PropertyFilter {
key: "country".to_string(),
value: Some(json!("US")),
operator: Some(OperatorType::Exact),
prop_type: PropertyType::Person,
group_type_index: None,
negation: None,
compiled_regex: None,
extra: Default::default(),
}]),
rollout_percentage: Some(50.0),
variant: None,
..Default::default()
}],
multivariate: None,
aggregation_group_type_index: None,
payloads: None,
feature_enrollment: None,
holdout: None,
early_exit: None,
extra: Default::default(),
},
deleted: false,
active: true,
ensure_experience_continuity: Some(false),
version: Some(1),
evaluation_runtime: Some("all".to_string()),
evaluation_tags: None,
bucketing_identifier: None,
},
FeatureFlag {
id: 2,
team_id: team.id,
name: Some("New User Interface".to_string()),
key: "new_ui".to_string(),
has_experiment: false,
filters: FlagFilters {
non_v1: None,
groups: vec![],
multivariate: None,
aggregation_group_type_index: None,
payloads: None,
feature_enrollment: None,
holdout: None,
early_exit: None,
extra: Default::default(),
},
deleted: false,
active: false,
ensure_experience_continuity: Some(false),
version: Some(1),
evaluation_runtime: Some("all".to_string()),
evaluation_tags: None,
bucketing_identifier: None,
},
FeatureFlag {
id: 3,
team_id: team.id,
name: Some("Premium Feature".to_string()),
key: "premium_feature".to_string(),
has_experiment: false,
filters: FlagFilters {
non_v1: None,
groups: vec![FlagPropertyGroup {
properties: Some(vec![PropertyFilter {
key: "is_premium".to_string(),
value: Some(json!(true)),
operator: Some(OperatorType::Exact),
prop_type: PropertyType::Person,
group_type_index: None,
negation: None,
compiled_regex: None,
extra: Default::default(),
}]),
rollout_percentage: Some(100.0),
variant: None,
..Default::default()
}],
multivariate: None,
aggregation_group_type_index: None,
payloads: None,
feature_enrollment: None,
holdout: None,
early_exit: None,
extra: Default::default(),
},
deleted: false,
active: true,
ensure_experience_continuity: Some(false),
version: Some(1),
evaluation_runtime: Some("all".to_string()),
evaluation_tags: None,
bucketing_identifier: None,
},
];
let evaluation_metadata = Arc::new(EvaluationMetadata::single_stage(&flags_vec));
let mock_flags = FeatureFlagList {
flags: PreparedFlags::seal(flags_vec),
evaluation_metadata,
..Default::default()
};
// Write to hypercache (new format: {"flags": [...]})
update_flags_in_hypercache(redis_client.clone(), team.id, &mock_flags, None)
.await
.expect("Failed to insert mock flags in hypercache");
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
// Test fetching from hypercache
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
assert!(result.is_ok());
let flag_result = result.unwrap();
assert!(!matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
));
assert_eq!(flag_result.prepared.flags.len(), mock_flags.flags.len());
// Verify the contents of the fetched flags
let beta_feature = flag_result
.prepared
.flags
.iter()
.find(|f| f.key == "beta_feature")
.unwrap();
assert!(beta_feature.active);
assert_eq!(
beta_feature.filters.groups[0].rollout_percentage,
Some(50.0)
);
assert_eq!(
beta_feature.filters.groups[0].properties.as_ref().unwrap()[0].key,
"country"
);
let new_ui = flag_result
.prepared
.flags
.iter()
.find(|f| f.key == "new_ui")
.unwrap();
assert!(!new_ui.active);
assert!(new_ui.filters.groups.is_empty());
let premium_feature = flag_result
.prepared
.flags
.iter()
.find(|f| f.key == "premium_feature")
.unwrap();
assert!(premium_feature.active);
assert_eq!(
premium_feature.filters.groups[0].rollout_percentage,
Some(100.0)
);
assert_eq!(
premium_feature.filters.groups[0]
.properties
.as_ref()
.unwrap()[0]
.key,
"is_premium"
);
}
#[tokio::test]
async fn test_get_flags_from_hypercache_compressed_payload() {
use common_compression::compress_zstd;
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert team in Redis");
// Create a large payload with multiple flags (>512 bytes triggers compression in Django)
let large_flags_vec: Vec<FeatureFlag> = (0..10)
.map(|i| FeatureFlag {
id: i,
team_id: team.id,
name: Some(format!("Test Flag {i} with a longer name for size")),
key: format!("test_flag_{i}_with_extra_chars_for_larger_payload"),
has_experiment: false,
deleted: false,
active: i % 2 == 0,
filters: FlagFilters {
non_v1: None,
groups: vec![FlagPropertyGroup {
properties: Some(vec![PropertyFilter {
key: format!("property_key_{i}"),
value: Some(serde_json::json!(format!("value_{i}"))),
operator: Some(OperatorType::Exact),
prop_type: PropertyType::Person,
group_type_index: None,
negation: None,
compiled_regex: None,
extra: Default::default(),
}]),
rollout_percentage: Some(50.0 + i as f64),
variant: None,
..Default::default()
}],
multivariate: None,
aggregation_group_type_index: None,
payloads: None,
feature_enrollment: None,
holdout: None,
early_exit: None,
extra: Default::default(),
},
ensure_experience_continuity: Some(false),
version: Some(1),
evaluation_runtime: Some("all".to_string()),
evaluation_tags: None,
bucketing_identifier: None,
})
.collect();
let large_flags = FeatureFlagList {
flags: PreparedFlags::seal(large_flags_vec),
..Default::default()
};
// Serialize exactly like Django does for large payloads: JSON -> Pickle -> Zstd
let wrapper = HypercacheFlagsWrapper {
flags: large_flags.flags.to_vec(),
evaluation_metadata: EvaluationMetadata::single_stage(&large_flags.flags),
cohorts: None,
};
let json_string = serde_json::to_string(&wrapper).expect("Failed to serialize to JSON");
// Verify payload is large enough to trigger compression
assert!(
json_string.len() > 512,
"Payload should be >512 bytes to simulate compressed data, was {} bytes",
json_string.len()
);
let pickled_bytes =
serde_pickle::to_vec(&json_string, Default::default()).expect("Failed to pickle");
let compressed_bytes = compress_zstd(&pickled_bytes).expect("Failed to compress with zstd");
// Write compressed data directly to Redis (simulating Django's HyperCache)
let cache_key = hypercache_test_key(team.id);
redis_client
.set_bytes(cache_key, compressed_bytes, None)
.await
.expect("Failed to write compressed data to Redis");
// Create FlagService and verify it can read the compressed data
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
assert!(
result.is_ok(),
"Failed to read compressed flags: {:?}",
result.err()
);
let flag_result = result.unwrap();
assert!(
!matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
),
"Expected cache hit for compressed data"
);
assert_eq!(
flag_result.prepared.flags.len(),
10,
"Expected 10 flags from compressed payload"
);
// Verify flag contents were correctly decompressed and parsed
let first_flag = &flag_result.prepared.flags[0];
assert_eq!(
first_flag.key,
"test_flag_0_with_extra_chars_for_larger_payload"
);
assert!(first_flag.active);
let last_flag = &flag_result.prepared.flags[9];
assert_eq!(
last_flag.key,
"test_flag_9_with_extra_chars_for_larger_payload"
);
assert!(!last_flag.active); // 9 % 2 != 0
}
#[tokio::test]
async fn test_get_flags_falls_back_to_pg_on_hypercache_miss() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
let dependency = context
.insert_flag(team.id, None)
.await
.expect("Failed to insert flag");
let dependent = context
.insert_flag(
team.id,
Some(crate::mock!(FeatureFlagRow,
team_id: team.id,
key: "dependent_flag".to_string(),
filters: json!({
"groups": [{
"properties": [{
"key": dependency.id.to_string(),
"type": "flag",
"value": true,
"operator": "flag_evaluates_to",
}],
"rollout_percentage": 100,
}],
}),
)),
)
.await
.expect("Failed to insert dependent flag");
// Don't populate hypercache - should fall back to PG
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let flag_result = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("PG fallback should succeed");
assert!(matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
));
assert_eq!(
flag_result.prepared.evaluation_metadata.dependency_stages,
vec![vec![dependency.id], vec![dependent.id]]
);
}
#[tokio::test]
async fn test_get_flags_falls_back_to_pg_on_redis_timeout() {
use common_redis::{CustomRedisError, MockRedisClient};
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
// Set up mock redis client to return Timeout on read
// HyperCacheReader calls get_raw_bytes(), so we mock that method
let mut mock_client = MockRedisClient::new();
mock_client.get_raw_bytes_ret(
&hypercache_test_key(team.id),
Err(CustomRedisError::Timeout),
);
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client.clone());
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
// Use a real Redis client for team hypercache since we're only mocking flags cache
let team_redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(team_redis_client).await;
let flag_service = FlagService::new(
redis_client,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
// Should succeed by falling back to PostgreSQL
assert!(result.is_ok());
let flag_result = result.unwrap();
assert!(matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
));
}
#[tokio::test]
async fn test_get_flags_falls_back_to_pg_on_redis_unavailable() {
use common_redis::{CustomRedisError, MockRedisClient, RedisErrorKind};
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
// Set up mock redis client to return Redis error (maps to RedisUnavailable)
// HyperCacheReader calls get_raw_bytes(), so we mock that method
let mut mock_client = MockRedisClient::new();
mock_client.get_raw_bytes_ret(
&hypercache_test_key(team.id),
Err(CustomRedisError::from_redis_kind(
RedisErrorKind::IoError,
"Connection refused",
)),
);
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client.clone());
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
// Use a real Redis client for team hypercache since we're only mocking flags cache
let team_redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(team_redis_client).await;
let flag_service = FlagService::new(
redis_client,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
// Should succeed by falling back to PostgreSQL
assert!(result.is_ok());
let flag_result = result.unwrap();
assert!(matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
));
// Verify SET was NOT called (cache write was skipped)
// This is tested via FlagsReadThroughCache behavior
let client_calls = mock_client.get_calls();
assert!(
!client_calls.iter().any(|call| call.op == "set"),
"Expected SET to NOT be called for RedisUnavailable error, but it was"
);
}
/// Verifies that the Rust service never writes to cache on PG fallback.
/// Django now handles cache population via HyperCache; Rust is read-only.
#[tokio::test]
async fn test_cache_write_never_performed_on_pg_fallback() {
use common_redis::MockRedisClient;
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
// Set up mock redis client with no return value for the key.
// MockRedisClient returns NotFound for keys with no mock set up,
// which triggers the PG fallback path.
let mock_client = MockRedisClient::new();
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client.clone());
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
let team_redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(team_redis_client).await;
let flag_service = FlagService::new(
redis_client,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
// Should succeed by falling back to PostgreSQL
assert!(result.is_ok());
let flag_result = result.unwrap();
assert!(
matches!(
flag_result.cache_source,
common_hypercache::CacheSource::Fallback
),
"Expected fallback to PostgreSQL since mock returned NotFound"
);
// Verify no SET calls were made - Django handles cache writes, not Rust
let client_calls = mock_client.get_calls();
assert!(
!client_calls
.iter()
.any(|call| call.op == "set" || call.op == "set_bytes"),
"Cache write detected after PG fallback. Rust should be read-only; \
Django handles cache population via HyperCache. Found calls: {client_calls:?}",
);
}
/// Corrupt Redis payload must hard-fail under `flag_data_parsing_error` rather
/// than silently fall back to PG.
#[tokio::test]
async fn test_get_flags_hard_fails_on_hypercache_parse_error() {
use common_redis::MockRedisClient;
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
// Pickle a string that isn't the sentinel and isn't valid JSON for the wrapper.
let invalid_json = "not valid json {{{";
let pickled =
serde_pickle::to_vec(&invalid_json, Default::default()).expect("Failed to pickle");
let mut mock_client = MockRedisClient::new();
mock_client.get_raw_bytes_ret(&hypercache_test_key(team.id), Ok(pickled));
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client);
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
let team_redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(team_redis_client).await;
let flag_service = FlagService::new(
redis_client,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
assert!(
matches!(
result,
Err(FlagError::InternalError {
code: "flag_data_parsing_error",
..
})
),
"parse error must hard-fail, got {result:?}"
);
}
/// One flag that the cache payload cannot parse must not fail the team's read.
/// The team keeps every other flag and gets no 500.
#[tokio::test]
async fn test_get_flags_skips_unparseable_flag_in_hypercache_payload() {
use crate::flags::test_helpers::payload_with_unparseable_flag;
use common_redis::MockRedisClient;
let team_id = 4242;
let payload = payload_with_unparseable_flag(team_id).to_string();
let pickled = serde_pickle::to_vec(&payload, Default::default()).expect("Failed to pickle");
let mut mock_client = MockRedisClient::new();
mock_client.get_raw_bytes_ret(&hypercache_test_key(team_id), Ok(pickled));
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client);
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
let team_hypercache_reader =
setup_team_hypercache_reader_with_mock_redis(redis_client.clone());
let flag_service = FlagService::new(
redis_client,
setup_pg_reader_client(None),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service
.get_flags_from_cache_or_pg(team_id)
.await
.expect("one unparseable flag must not fail the team's read");
let keys: Vec<&str> = result
.prepared
.flags
.iter()
.map(|f| f.key.as_str())
.collect();
assert_eq!(keys, vec!["healthy_flag", "dependent_flag"]);
assert_eq!(
result.prepared.evaluation_metadata.dependency_stages,
vec![vec![1], vec![3]]
);
}
/// Sibling of the Json parse-error test: guards the `Pickle` arm of the
/// `Err(e @ (Json(_) | Pickle(_)))` pattern against accidental narrowing.
#[tokio::test]
async fn test_get_flags_hard_fails_on_hypercache_pickle_error() {
use common_redis::MockRedisClient;
let context = TestContext::new(None).await;
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team");
// Raw non-pickle bytes make serde_pickle::from_slice::<String> fail.
let non_pickle_bytes = b"this is not pickle data".to_vec();
let mut mock_client = MockRedisClient::new();
mock_client.get_raw_bytes_ret(&hypercache_test_key(team.id), Ok(non_pickle_bytes));
let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client);
let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
let team_redis_client = setup_redis_client(None).await;
let team_hypercache_reader = setup_team_hypercache_reader(team_redis_client).await;
let flag_service = FlagService::new(
redis_client,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
NegativeCache::new(100, 300),
false,
);
let result = flag_service.get_flags_from_cache_or_pg(team.id).await;
assert!(
matches!(
result,
Err(FlagError::InternalError {
code: "flag_data_parsing_error",
..
})
),
"pickle error must hard-fail, got {result:?}"
);
}
#[tokio::test]
async fn test_negative_cache_returns_error_for_cached_invalid_token() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let negative_cache = NegativeCache::new(100, 300);
negative_cache.insert("known_bad_token".to_string());
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache,
false,
);
let result = flag_service
.verify_token_and_get_team("known_bad_token")
.await;
assert!(matches!(result, Err(FlagError::TokenValidationError)));
}
#[tokio::test]
async fn test_negative_cache_populated_on_invalid_token() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let negative_cache = NegativeCache::new(100, 300);
assert!(!negative_cache.contains("nonexistent_token"));
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache.clone(),
false,
);
// First call should fail and populate the negative cache
let result = flag_service
.verify_token_and_get_team("nonexistent_token")
.await;
assert!(matches!(result, Err(FlagError::TokenValidationError)));
assert!(negative_cache.contains("nonexistent_token"));
}
#[tokio::test]
async fn test_negative_cache_does_not_affect_valid_tokens() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert team");
let negative_cache = NegativeCache::new(100, 300);
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache.clone(),
false,
);
// Valid token should succeed and not be added to negative cache
let result = flag_service
.verify_token_and_get_team(&team.api_token)
.await;
assert!(result.is_ok());
assert!(!negative_cache.contains(&team.api_token));
}
/// Verifies the full negative cache lifecycle: the first call for an invalid
/// token hits the backend and populates the cache, and the second call is
/// served from cache without making additional Redis calls.
#[tokio::test]
async fn test_second_call_for_invalid_token_skips_backends() {
use common_redis::MockRedisClient;
let mock_client = MockRedisClient::new();
let mock_redis: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client.clone());
let team_hypercache_reader = setup_hypercache_reader_with_mock_redis(mock_redis.clone());
let hypercache_reader = setup_hypercache_reader_with_mock_redis(mock_redis.clone());
let context = TestContext::new(None).await;
let negative_cache = NegativeCache::new(100, 300);
let flag_service = FlagService::new(
mock_redis,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache.clone(),
false,
);
// First call: misses cache, hits Redis (mock) + PG fallback, fails, populates negative cache
let result = flag_service.verify_token_and_get_team("bad_token").await;
assert!(matches!(result, Err(FlagError::TokenValidationError)));
assert!(negative_cache.contains("bad_token"));
let calls_after_first = mock_client.get_calls().len();
assert!(
calls_after_first > 0,
"First call should have made Redis calls"
);
// Second call: should hit negative cache and return immediately
let result = flag_service.verify_token_and_get_team("bad_token").await;
assert!(matches!(result, Err(FlagError::TokenValidationError)));
let calls_after_second = mock_client.get_calls().len();
assert_eq!(
calls_after_first, calls_after_second,
"Second call should not make additional Redis calls (served from negative cache)"
);
}
#[tokio::test]
async fn test_skip_pg_fallback_still_resolves_token_from_hypercache() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("Failed to insert new team in Redis");
let negative_cache = NegativeCache::new(100, 300);
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache.clone(),
true, // skip PG fallback
);
// Token is in HyperCache, so lookup should succeed even with PG fallback disabled
let result = flag_service
.verify_token_and_get_team(&team.api_token)
.await;
assert!(result.is_ok());
assert_eq!(result.unwrap().api_token, team.api_token);
assert!(
!negative_cache.contains(&team.api_token),
"Valid token found in HyperCache should not be in negative cache"
);
}
#[rstest]
#[case::skip_enabled_rejects(true)]
#[case::skip_disabled_falls_back_to_pg(false)]
#[tokio::test]
async fn test_skip_pg_fallback_with_pg_only_token(#[case] skip_pg: bool) {
use common_redis::MockRedisClient;
let mock_client = MockRedisClient::new();
let mock_redis: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client.clone());
let team_hypercache_reader =
setup_team_hypercache_reader_with_mock_redis(mock_redis.clone());
let hypercache_reader = setup_hypercache_reader_with_mock_redis(mock_redis.clone());
let context = TestContext::new(None).await;
// Team exists in PG but not in HyperCache
let team = context
.insert_new_team(None)
.await
.expect("Failed to insert team in PG");
let negative_cache = NegativeCache::new(100, 300);
let flag_service = FlagService::new(
mock_redis,
context.non_persons_reader.clone(),
team_hypercache_reader,
hypercache_reader,
Arc::new(FlagDefinitionsCache::disabled()),
negative_cache.clone(),
skip_pg,
);
let result = flag_service
.verify_token_and_get_team(&team.api_token)
.await;
if skip_pg {
assert!(matches!(result, Err(FlagError::TokenValidationError)));
assert!(
negative_cache.contains(&team.api_token),
"Token should be negative-cached when PG fallback is skipped"
);
} else {
let resolved_team = result.expect("Should resolve via PG fallback");
assert_eq!(resolved_team.api_token, team.api_token);
assert!(
!negative_cache.contains(&team.api_token),
"Valid token should not be in negative cache"
);
}
}
// =========================================================================
// FlagDefinitionsCache integration: exercises the real (non-disabled) cache
// through FlagService so the Arc-share path and etag-based invalidation
// survive the full hypercache round-trip (serde_json → pickle → Redis →
// get_etag → moka). Unit tests in flag_definitions_cache cover the cache
// in isolation; these pin the service boundary.
// =========================================================================
/// Builds a single-flag wrapper with a Regex operator so the cache hit path
/// has to pre-compile an actual fancy_regex pattern.
fn single_regex_flag_wrapper(team_id: i32, pattern: &str) -> FeatureFlagList {
use crate::mock;
use crate::properties::property_models::PropertyFilter;
use crate::utils::mock::MockInto;
let flag = mock!(FeatureFlag,
team_id: team_id,
name: "Regex Flag".mock_into(),
key: "regex_flag".mock_into(),
filters: mock!(PropertyFilter,
key: "email".mock_into(),
value: Some(json!(pattern)),
operator: Some(OperatorType::Regex),
)
.mock_into(),
);
let evaluation_metadata = Arc::new(EvaluationMetadata::single_stage(std::slice::from_ref(
&flag,
)));
FeatureFlagList {
flags: PreparedFlags::seal(vec![flag]),
evaluation_metadata,
..Default::default()
}
}
fn real_cache() -> Arc<FlagDefinitionsCache> {
// Default 128 MB / 90 s — same as production. Matters here only insofar
// as entries must not be evicted between the two fetches in each test.
Arc::new(FlagDefinitionsCache::new(None, None))
}
/// End-to-end Arc sharing: two `get_flags_from_cache_or_pg` calls for the
/// same team, against identical hypercache content, must return the same
/// `Arc<PreparedFlagDefinitions>`. Regressions here (e.g. a stray clone in
/// FlagService or an etag fetch that shifts between calls) would restore
/// per-request regex compilation without failing any existing test.
#[tokio::test]
async fn test_flag_service_cache_hit_reuses_arc() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("insert team");
let mock_flags = single_regex_flag_wrapper(team.id, r"^user@.*\.com$");
update_flags_in_hypercache(redis_client.clone(), team.id, &mock_flags, None)
.await
.expect("write hypercache");
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
setup_team_hypercache_reader(redis_client.clone()).await,
setup_hypercache_reader(redis_client.clone()).await,
real_cache(),
NegativeCache::new(100, 300),
false,
);
let first = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("first fetch");
let second = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("second fetch");
assert!(
Arc::ptr_eq(&first.prepared, &second.prepared),
"second fetch must return the cached Arc, not recompile"
);
// The cached path must produce compiled regex — otherwise evaluation
// falls back to on-the-fly compilation and the caching is useless.
let re = &first.prepared.flags[0].filters.groups[0]
.properties
.as_ref()
.unwrap()[0]
.compiled_regex;
assert!(
matches!(
re,
Some(crate::properties::property_models::CompiledRegex::Compiled(
_
))
),
"cached flag must carry a compiled regex, got {re:?}"
);
}
/// A hypercache rewrite with different content must produce a fresh Arc:
/// Django writes a new etag whenever the payload bytes change, so the
/// `(team_id, etag)` cache key shifts and the next fetch misses + recompiles.
/// Guards against an etag computed over something more abstract than the
/// actual flag content — e.g. only over flag IDs.
#[tokio::test]
async fn test_flag_service_content_change_invalidates_cache() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("insert team");
update_flags_in_hypercache(
redis_client.clone(),
team.id,
&single_regex_flag_wrapper(team.id, r"^v1@.*$"),
None,
)
.await
.expect("write v1");
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
setup_team_hypercache_reader(redis_client.clone()).await,
setup_hypercache_reader(redis_client.clone()).await,
real_cache(),
NegativeCache::new(100, 300),
false,
);
let v1 = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("v1 fetch");
// Rewrite the same flag id with a different regex pattern.
update_flags_in_hypercache(
redis_client.clone(),
team.id,
&single_regex_flag_wrapper(team.id, r"^v2@.*$"),
None,
)
.await
.expect("write v2");
let v2 = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("v2 fetch");
assert!(
!Arc::ptr_eq(&v1.prepared, &v2.prepared),
"content change must produce a new Arc (etag must shift)"
);
let re_v2 = &v2.prepared.flags[0].filters.groups[0]
.properties
.as_ref()
.unwrap()[0]
.compiled_regex;
match re_v2 {
Some(crate::properties::property_models::CompiledRegex::Compiled(r)) => {
assert_eq!(
r.as_str(),
r"^v2@.*$",
"v2 fetch must carry the v2 pattern, not a stale compile"
);
}
other => panic!("expected compiled v2 regex, got {other:?}"),
}
}
/// Etag stability across the Django-path round-trip: write, fetch, rewrite
/// byte-identical content, fetch again — must return the same Arc. The
/// guarantee we lean on for the version-key fast path is that identical
/// payloads produce identical etags (sha256 over `json.dumps(data,
/// sort_keys=True)` is deterministic on Django's side; the test helper
/// uses `compute_etag` over the same serialized bytes). Without this, the
/// etag would shift on every cache refresh and the in-memory cache would
/// effectively always miss.
#[tokio::test]
async fn test_identical_content_round_trip_keeps_etag_stable() {
let redis_client = setup_redis_client(None).await;
let pg_client = setup_pg_reader_client(None);
let team = insert_new_team_in_redis(redis_client.clone())
.await
.expect("insert team");
let flags = single_regex_flag_wrapper(team.id, r"^user@.*\.com$");
// First write + fetch.
update_flags_in_hypercache(redis_client.clone(), team.id, &flags, None)
.await
.expect("write #1");
let flag_service = FlagService::new(
redis_client.clone(),
pg_client.clone(),
setup_team_hypercache_reader(redis_client.clone()).await,
setup_hypercache_reader(redis_client.clone()).await,
real_cache(),
NegativeCache::new(100, 300),
false,
);
let first = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("first fetch");
// Rewrite identical content. This exercises a fresh pickle+etag path
// producing the same etag bytes — the in-memory cache must still hit.
update_flags_in_hypercache(redis_client.clone(), team.id, &flags, None)
.await
.expect("write #2");
let second = flag_service
.get_flags_from_cache_or_pg(team.id)
.await
.expect("second fetch after identical rewrite");
assert!(
Arc::ptr_eq(&first.prepared, &second.prepared),
"identical content re-written must produce the same etag and reuse the cached Arc"
);
}
}