Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): skip an unparseable flag instead of failing the team read (#98903)

posthog[bot]f58e1bf384bc
diff --git a/docs/internal/feature-flags/hypercache-system.md b/docs/internal/feature-flags/hypercache-system.md
index d52c22ccf6ad..ebddf2d3b593 100644
--- a/docs/internal/feature-flags/hypercache-system.md
+++ b/docs/internal/feature-flags/hypercache-system.md
@@ -131,6 +131,7 @@ A referenced entry is load-bearing: the matcher pre-seeds its id as false, so a
The service's PostgreSQL fallback runs the same classification, so a Redis miss serves the flag set the cache would have carried.
Both builders run the same fixture, `rust/feature-flags/tests/fixtures/flags_cache_config_formats.json`.
One known divergence remains: Rust decodes v1 documents into typed structs, so an evaluable row it cannot decode (a property filter without a `type` key, for example) is omitted there with its dependents, while Python, which only reads the cohort and flag references, still publishes it.
+When the Rust service reads a Python-written entry that holds such a row, it skips that one flag, counts it in `flags_flag_malformed_filter_total`, and marks every flag that depends on it as missing a dependency, so the team's other flags still evaluate.
`verify_team_flags` reports and repairs such a team, attributed to the writer that produced it.
An entry written before this rule can hold an inactive non-v1 row already blanked to `{"groups": []}`; its cached bytes cannot recover the original format, so the entry is rebuilt from the database rows by the next invalidation, refresh sweep, or verifier repair rather than reclassified in place.
Until then an unreferenced blanked row is inert for the matcher, and an active dependent of one reports as `STALE_IN_CACHE` and triggers the repair.
diff --git a/rust/feature-flags/src/flags/cache_builder.rs b/rust/feature-flags/src/flags/cache_builder.rs
index fff7a1845eeb..9e0d3cc74e00 100644
--- a/rust/feature-flags/src/flags/cache_builder.rs
+++ b/rust/feature-flags/src/flags/cache_builder.rs
@@ -256,6 +256,7 @@ pub fn compute_flag_dependencies(flags: &[FeatureFlag]) -> Result<EvaluationMeta
dependency_stages: result.stages,
flags_with_missing_deps: result.nodes_with_missing_deps,
transitive_deps,
+ ..Default::default()
})
}
diff --git a/rust/feature-flags/src/flags/cache_shadow.rs b/rust/feature-flags/src/flags/cache_shadow.rs
index 41cf132d2d31..cb69c6bafd47 100644
--- a/rust/feature-flags/src/flags/cache_shadow.rs
+++ b/rust/feature-flags/src/flags/cache_shadow.rs
@@ -1010,12 +1010,14 @@ mod tests {
dependency_stages: vec![vec![1]],
flags_with_missing_deps: vec![],
transitive_deps: [(1, Default::default())].into(),
+ ..Default::default()
};
let mut cached = live(vec![flag_from_json(base_flag_json(1))]);
cached.evaluation_metadata = Some(EvaluationMetadata {
dependency_stages: vec![vec![1]],
flags_with_missing_deps: vec![1],
transitive_deps: [(1, Default::default())].into(),
+ ..Default::default()
});
let diffs = diff_live_entry(&built, &cached);
@@ -1035,6 +1037,7 @@ mod tests {
dependency_stages: vec![vec![1, 2]],
flags_with_missing_deps: vec![],
transitive_deps: [(1, Default::default()), (2, Default::default())].into(),
+ ..Default::default()
};
let mut built = wrapper(vec![]);
built.evaluation_metadata = metadata();
diff --git a/rust/feature-flags/src/flags/feature_flag_list.rs b/rust/feature-flags/src/flags/feature_flag_list.rs
index 85cac0828410..c473f59fcb05 100644
--- a/rust/feature-flags/src/flags/feature_flag_list.rs
+++ b/rust/feature-flags/src/flags/feature_flag_list.rs
@@ -11,7 +11,11 @@ use crate::metrics::consts::{
use common_database::PostgresReader;
use common_types::TeamId;
use metrics::counter;
+use serde::de::Deserializer;
+use serde::Deserialize;
+use serde_json::value::RawValue;
use std::collections::HashMap;
+use std::collections::HashSet;
use std::sync::Arc;
/// Parsed hypercache result: flags, evaluation metadata, optional preloaded cohorts.
@@ -69,6 +73,98 @@ impl From<Vec<FeatureFlag>> for PreparedFlags {
}
}
+fn record_dropped_flags(dropped: u64) {
+ counter!(FLAG_MALFORMED_FILTER_COUNTER).increment(dropped);
+ counter!(FLAG_MALFORMED_FILTER_READ_COUNTER).increment(1);
+}
+
+/// A flags array that skips an element it cannot deserialize, instead of
+/// failing the whole array. Keeps the key and parse error of each skipped flag,
+/// because the caller knows the team id and the deserializer does not.
+#[derive(Debug)]
+pub struct LenientFlags {
+ flags: Vec<FeatureFlag>,
+ skipped: Vec<(String, serde_json::Error)>,
+}
+
+impl<'de> Deserialize<'de> for LenientFlags {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ // This first splits the array into one borrowed JSON slice per flag, because
+ // serde cannot resume a sequence after one element fails. A borrowed RawValue
+ // fails unless the deserializer reads a string or byte slice, as the
+ // hypercache reader's `serde_json::from_str` does.
+ let raw_flags: Vec<&'de RawValue> = Vec::deserialize(deserializer)?;
+
+ let mut flags = Vec::with_capacity(raw_flags.len());
+ let mut skipped = Vec::new();
+ for raw in &raw_flags {
+ match serde_json::from_str::<FeatureFlag>(raw.get()) {
+ Ok(flag) => flags.push(flag),
+ Err(e) => {
+ let flag_key = serde_json::from_str::<serde_json::Value>(raw.get())
+ .ok()
+ .and_then(|v| v["key"].as_str().map(str::to_owned))
+ .unwrap_or_else(|| "<unknown>".to_owned());
+ skipped.push((flag_key, e));
+ }
+ }
+ }
+ Ok(LenientFlags { flags, skipped })
+ }
+}
+
+/// The hypercache payload as the serving read parses it. The contract test
+/// parses its fixture with this type and with the strict default, so a flag
+/// shape that Rust cannot read fails that test instead of being skipped silently.
+pub type LenientHypercacheFlagsWrapper = HypercacheFlagsWrapper<LenientFlags>;
+
+impl LenientHypercacheFlagsWrapper {
+ /// Drops the flags that failed to parse, and every metadata reference to them.
+ pub fn into_wrapper(self, team_id: TeamId) -> HypercacheFlagsWrapper {
+ let HypercacheFlagsWrapper {
+ flags,
+ mut evaluation_metadata,
+ cohorts,
+ } = self;
+
+ if !flags.skipped.is_empty() {
+ for (flag_key, e) in &flags.skipped {
+ tracing::warn!(
+ "Failed to deserialize cached flag {} in team {}: {}",
+ flag_key,
+ team_id,
+ e
+ );
+ }
+ let dropped = flags.skipped.len() as u64;
+ record_dropped_flags(dropped);
+ // Details (team_id, flag_key) are logged above to avoid high-cardinality labels
+ counter!(
+ TOMBSTONE_COUNTER,
+ "namespace" => "feature_flags",
+ "operation" => "hypercache_flag_deserialization_error",
+ "component" => "feature_flag_list",
+ )
+ .increment(dropped);
+
+ evaluation_metadata.has_unparseable_flags = true;
+ // Django computes the metadata over every flag, so it still points at
+ // the flags we just skipped.
+ let kept_flag_ids: HashSet<i32> = flags.flags.iter().map(|f| f.id).collect();
+ evaluation_metadata.retain_flags(&kept_flag_ids);
+ }
+
+ HypercacheFlagsWrapper {
+ flags: flags.flags,
+ evaluation_metadata,
+ cohorts,
+ }
+ }
+}
+
impl FeatureFlagList {
pub fn new(flags: Vec<FeatureFlag>) -> Self {
Self {
@@ -277,8 +373,7 @@ impl FeatureFlagList {
.collect();
if !undecodable.is_empty() {
- counter!(FLAG_MALFORMED_FILTER_COUNTER).increment(undecodable.len() as u64);
- counter!(FLAG_MALFORMED_FILTER_READ_COUNTER).increment(1);
+ record_dropped_flags(undecodable.len() as u64);
}
tracing::debug!(
@@ -882,6 +977,40 @@ mod tests {
use serde_json::json;
+ #[test]
+ fn test_lenient_wrapper_skips_unparseable_flag_and_keeps_the_rest() {
+ use crate::flags::test_helpers::payload_with_unparseable_flag;
+ use crate::utils::test_utils::counter_total;
+ use metrics_util::debugging::DebuggingRecorder;
+
+ let recorder = DebuggingRecorder::new();
+ let snapshotter = recorder.snapshotter();
+ let _guard = metrics::set_default_local_recorder(&recorder);
+
+ let wrapper: LenientHypercacheFlagsWrapper =
+ serde_json::from_str(&payload_with_unparseable_flag(123).to_string()).unwrap();
+
+ let wrapper = wrapper.into_wrapper(123);
+ let metadata = &wrapper.evaluation_metadata;
+
+ let keys: Vec<&str> = wrapper.flags.iter().map(|f| f.key.as_str()).collect();
+ assert_eq!(keys, vec!["healthy_flag", "dependent_flag"]);
+ assert_eq!(metadata.dependency_stages, vec![vec![1], vec![3]]);
+ assert_eq!(metadata.flags_with_missing_deps, vec![1, 3]);
+ assert!(metadata.has_unparseable_flags);
+ assert!(!metadata.transitive_deps.contains_key(&2));
+ assert!(metadata.transitive_deps[&3].is_empty());
+
+ assert_eq!(
+ counter_total(&snapshotter, FLAG_MALFORMED_FILTER_COUNTER, &[]),
+ 1
+ );
+ assert_eq!(
+ counter_total(&snapshotter, FLAG_MALFORMED_FILTER_READ_COUNTER, &[]),
+ 1
+ );
+ }
+
#[test]
fn test_from_wrapper_valid_flags() {
let wrapper: HypercacheFlagsWrapper = serde_json::from_value(json!({
@@ -1343,6 +1472,14 @@ mod tests {
// Verify flags parsed correctly
assert_eq!(flags.len(), 7, "Expected 7 flags in contract fixture");
+ let served: LenientHypercacheFlagsWrapper = serde_json::from_str(fixture)
+ .expect("The serving read failed to deserialize hypercache_contract.json");
+ assert_eq!(
+ served.into_wrapper(99).flags.len(),
+ 7,
+ "The serving read skipped a flag that the contract fixture holds"
+ );
+
// Full flag with all optional fields
let full_flag = &flags[0];
assert_eq!(full_flag.key, "full-flag");
diff --git a/rust/feature-flags/src/flags/flag_definitions_cache.rs b/rust/feature-flags/src/flags/flag_definitions_cache.rs
index 61cf7391c8fa..39f410968acd 100644
--- a/rust/feature-flags/src/flags/flag_definitions_cache.rs
+++ b/rust/feature-flags/src/flags/flag_definitions_cache.rs
@@ -647,6 +647,7 @@ mod tests {
dependency_stages: vec![],
flags_with_missing_deps: vec![],
transitive_deps: Default::default(),
+ ..Default::default()
},
cohorts: None,
};
diff --git a/rust/feature-flags/src/flags/flag_models.rs b/rust/feature-flags/src/flags/flag_models.rs
index 50cdf62ee09c..60ab3d917eb4 100644
--- a/rust/feature-flags/src/flags/flag_models.rs
+++ b/rust/feature-flags/src/flags/flag_models.rs
@@ -80,6 +80,11 @@ pub struct EvaluationMetadata {
serialize_with = "serialize_string_keyed_i32_map"
)]
pub transitive_deps: HashMap<i32, HashSet<i32>>,
+ /// True when the serving read skipped a cached flag that it could not parse.
+ /// The response then sets `errorsWhileComputingFlags`, so an SDK merges the
+ /// response into its stored flags and keeps the skipped flag's last value.
+ #[serde(skip)]
+ pub has_unparseable_flags: bool,
}
impl EvaluationMetadata {
@@ -92,6 +97,33 @@ impl EvaluationMetadata {
..Default::default()
}
}
+
+ /// Drops every reference to a flag outside `kept_flag_ids`, and marks a kept
+ /// flag that depends on a dropped one as missing a dependency, so it fails
+ /// closed like any other flag whose dependency Django cannot resolve.
+ /// Changes nothing when `kept_flag_ids` holds every flag id that the metadata names.
+ pub fn retain_flags(&mut self, kept_flag_ids: &HashSet<i32>) {
+ self.flags_with_missing_deps
+ .retain(|id| kept_flag_ids.contains(id));
+ self.flags_with_missing_deps.extend(
+ self.transitive_deps
+ .iter()
+ .filter(|(id, deps)| kept_flag_ids.contains(id) && !deps.is_subset(kept_flag_ids))
+ .map(|(id, _)| *id),
+ );
+ self.flags_with_missing_deps.sort_unstable();
+ self.flags_with_missing_deps.dedup();
+
+ self.transitive_deps
+ .retain(|id, _| kept_flag_ids.contains(id));
+ for deps in self.transitive_deps.values_mut() {
+ deps.retain(|dep| kept_flag_ids.contains(dep));
+ }
+ for stage in &mut self.dependency_stages {
+ stage.retain(|id| kept_flag_ids.contains(id));
+ }
+ self.dependency_stages.retain(|stage| !stage.is_empty());
+ }
}
/// Wrapper struct for deserializing hypercache format:
@@ -105,9 +137,12 @@ impl EvaluationMetadata {
/// `_get_feature_flags_for_service()` in products/feature_flags/backend/flags_cache.py.
/// Field changes must follow the expand-and-contract pattern — see contract tests in
/// products/feature_flags/backend/test/test_flags_cache.py.
+///
+/// `F` is the flags array type. The serving read uses
+/// [`LenientFlags`](crate::flags::feature_flag_list::LenientFlags) to skip a flag it cannot read.
#[derive(Debug, Clone, Deserialize, Serialize)]
-pub struct HypercacheFlagsWrapper {
- pub flags: Vec<FeatureFlag>,
+pub struct HypercacheFlagsWrapper<F = Vec<FeatureFlag>> {
+ pub flags: F,
pub evaluation_metadata: EvaluationMetadata,
/// Cohort definitions referenced by flags (including transitive deps).
/// Precomputed by Django at cache-write time so the Rust service can skip
@@ -731,6 +766,7 @@ mod mock_impls {
dependency_stages: vec![],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::new(),
+ ..Default::default()
}
}
}
diff --git a/rust/feature-flags/src/flags/flag_service.rs b/rust/feature-flags/src/flags/flag_service.rs
index fd49e632004c..0e80d2a721d9 100644
--- a/rust/feature-flags/src/flags/flag_service.rs
+++ b/rust/feature-flags/src/flags/flag_service.rs
@@ -1,6 +1,7 @@
use crate::{
api::errors::FlagError,
flags::{
+ feature_flag_list::LenientHypercacheFlagsWrapper,
flag_definitions_cache::FlagDefinitionsCache,
flag_models::{HypercacheFlagsWrapper, PreparedFlagDefinitions},
},
@@ -179,8 +180,9 @@ impl FlagService {
}
/// Fetches flags from the hypercache (Redis → S3), falling back to PostgreSQL
- /// on cache miss or infra errors. Parse errors (`Json`/`Pickle`) hard-fail with
- /// a tombstone rather than silently serving PG data.
+ /// on cache miss or infra errors. A payload-level parse error (`Json`/`Pickle`)
+ /// hard-fails with a tombstone rather than silently serving PG data. A single
+ /// flag that does not parse is skipped, not fatal.
///
/// On the hot path the in-memory `FlagDefinitionsCache` is keyed on the etag
/// Django writes alongside the payload (`enable_etag=True`), so an in-memory
@@ -235,8 +237,10 @@ impl FlagService {
/// Hypercache-then-PG payload fetch, extracted so the in-memory cache can
/// invoke it lazily inside `get_or_load`. Returns `None` for the
/// `__missing__` sentinel (team has no flags), `Some(wrapper)` otherwise.
- /// `Json`/`Pickle` parse errors hard-fail with a tombstone — we never want
- /// to silently degrade to PG on data corruption.
+ /// A payload-level `Json`/`Pickle` parse error hard-fails with a tombstone,
+ /// because we never want to silently degrade to PG on data corruption. A flag
+ /// whose own JSON does not parse is skipped and counted instead, so one bad
+ /// flag does not fail the evaluation of the team's other flags.
async fn fetch_wrapper_or_pg(
&self,
team_id: TeamId,
@@ -245,10 +249,10 @@ impl FlagService {
match self
.flags_hypercache_reader
- .get_typed_with_source::<HypercacheFlagsWrapper>(&key)
+ .get_typed_with_source::<LenientHypercacheFlagsWrapper>(&key)
.await
{
- Ok((data, source)) => Ok((data, source)),
+ Ok((data, source)) => Ok((data.map(|w| w.into_wrapper(team_id)), source)),
Err(e @ (HyperCacheError::Json(_) | HyperCacheError::Pickle(_))) => {
counter!(
TOMBSTONE_COUNTER,
@@ -1017,6 +1021,54 @@ mod tests {
);
}
+ /// One flag that the cache payload cannot parse must not fail the team's read.
+ /// The team keeps every other flag and gets no 500.
+ #[tokio::test]
+ async fn test_get_flags_skips_unparseable_flag_in_hypercache_payload() {
+ use crate::flags::test_helpers::payload_with_unparseable_flag;
+ use common_redis::MockRedisClient;
+
+ let team_id = 4242;
+
+ let payload = payload_with_unparseable_flag(team_id).to_string();
+ let pickled = serde_pickle::to_vec(&payload, Default::default()).expect("Failed to pickle");
+
+ let mut mock_client = MockRedisClient::new();
+ mock_client.get_raw_bytes_ret(&hypercache_test_key(team_id), Ok(pickled));
+
+ let redis_client: Arc<dyn RedisClient + Send + Sync> = Arc::new(mock_client);
+ let hypercache_reader = setup_hypercache_reader_with_mock_redis(redis_client.clone());
+ let team_hypercache_reader =
+ setup_team_hypercache_reader_with_mock_redis(redis_client.clone());
+
+ let flag_service = FlagService::new(
+ redis_client,
+ setup_pg_reader_client(None),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+
+ let result = flag_service
+ .get_flags_from_cache_or_pg(team_id)
+ .await
+ .expect("one unparseable flag must not fail the team's read");
+
+ let keys: Vec<&str> = result
+ .prepared
+ .flags
+ .iter()
+ .map(|f| f.key.as_str())
+ .collect();
+ assert_eq!(keys, vec!["healthy_flag", "dependent_flag"]);
+ assert_eq!(
+ result.prepared.evaluation_metadata.dependency_stages,
+ vec![vec![1], vec![3]]
+ );
+ }
+
/// Sibling of the Json parse-error test: guards the `Pickle` arm of the
/// `Err(e @ (Json(_) | Pickle(_)))` pattern against accidental narrowing.
#[tokio::test]
diff --git a/rust/feature-flags/src/flags/test_flag_matching.rs b/rust/feature-flags/src/flags/test_flag_matching.rs
index efa17fe31448..8f3dd761c931 100644
--- a/rust/feature-flags/src/flags/test_flag_matching.rs
+++ b/rust/feature-flags/src/flags/test_flag_matching.rs
@@ -8516,6 +8516,7 @@ mod tests {
dependency_stages: vec![vec![1, 2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::new())]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -8609,6 +8610,7 @@ mod tests {
dependency_stages: vec![vec![1], vec![2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::from([1]))]),
+ ..Default::default()
}),
..Default::default()
};
@@ -8714,6 +8716,7 @@ mod tests {
dependency_stages: vec![vec![1], vec![2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::from([1]))]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -8813,6 +8816,7 @@ mod tests {
dependency_stages: vec![vec![1], vec![2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::from([1]))]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -9009,6 +9013,7 @@ mod tests {
(2, HashSet::from([1])),
(3, HashSet::from([1, 2])),
]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -10247,6 +10252,7 @@ mod tests {
(2, HashSet::from([3])),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -10336,6 +10342,7 @@ mod tests {
dependency_stages: vec![vec![1, 2]],
flags_with_missing_deps: vec![1],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::new())]),
+ ..Default::default()
}),
..Default::default()
};
diff --git a/rust/feature-flags/src/flags/test_helpers.rs b/rust/feature-flags/src/flags/test_helpers.rs
index 6368faea214b..e2b04d0af11c 100644
--- a/rust/feature-flags/src/flags/test_helpers.rs
+++ b/rust/feature-flags/src/flags/test_helpers.rs
@@ -2,6 +2,7 @@
use std::sync::Arc;
+use crate::utils::test_utils::unparseable_flag_json;
use crate::{
api::errors::{simplify_serde_error, FlagError},
flags::{
@@ -39,6 +40,37 @@ pub fn v2_filters_referencing(
filters
}
+/// A hypercache payload in which only flag 2 fails to deserialize. Flag 3 depends on flag 2.
+/// Flag 1 starts in `flags_with_missing_deps` so that a test sees whether a kept flag stays marked.
+pub fn payload_with_unparseable_flag(team_id: TeamId) -> serde_json::Value {
+ serde_json::json!({
+ "flags": [
+ {
+ "id": 1,
+ "key": "healthy_flag",
+ "team_id": team_id,
+ "active": true,
+ "deleted": false,
+ "filters": { "groups": [] }
+ },
+ unparseable_flag_json(2, "malformed_flag", team_id),
+ {
+ "id": 3,
+ "key": "dependent_flag",
+ "team_id": team_id,
+ "active": true,
+ "deleted": false,
+ "filters": { "groups": [] }
+ }
+ ],
+ "evaluation_metadata": {
+ "dependency_stages": [[1, 2], [3]],
+ "flags_with_missing_deps": [1, 2],
+ "transitive_deps": { "1": [], "2": [], "3": [2] }
+ }
+ })
+}
+
/// Generate the Django-compatible hypercache key for tests
/// Format: posthog:1:cache/teams/{team_id}/feature_flags/flags.json
/// The "posthog:1:" prefix matches Django's cache versioning
diff --git a/rust/feature-flags/src/handler/flags.rs b/rust/feature-flags/src/handler/flags.rs
index 3251cab00b55..8d964ff26601 100644
--- a/rust/feature-flags/src/handler/flags.rs
+++ b/rust/feature-flags/src/handler/flags.rs
@@ -353,62 +353,67 @@ pub async fn evaluate_for_request(
return Ok(FlagsResponse::new(false, HashMap::new(), None, request_id));
}
- if filtered_flags.flags.is_empty() {
- return Ok(FlagsResponse::new(false, HashMap::new(), None, request_id));
- }
-
- // Every flag is in the filter set (inactive, deleted, runtime/tag mismatch) — nothing to evaluate.
- // This is O(1) since the filter set includes deleted flags.
- if filtered_flags.filtered_out_flag_ids.len() >= filtered_flags.flags.len() {
- return Ok(FlagsResponse::new(false, HashMap::new(), None, request_id));
- }
+ let has_unparseable_flags = filtered_flags.evaluation_metadata.has_unparseable_flags;
- let persons_db_deadline = state
- .config
- .persons_db_deadline(tokio::time::Instant::now(), Some(request_entered_at));
- with_canonical_log(|log| {
- log.persons_db_deadline_bound = persons_db_deadline.map(|deadline| deadline.bound)
- });
-
- let ctx = FeatureFlagEvaluationContext {
- team_id,
- team_timezone,
- distinct_id,
- device_id,
- feature_flags: filtered_flags,
- persons_reader: state.database_pools.persons_reader.clone(),
- persons_writer: state.database_pools.persons_writer.clone(),
- non_persons_reader: state.database_pools.non_persons_reader.clone(),
- non_persons_writer: state.database_pools.non_persons_writer.clone(),
- cohort_cache: state.cohort_cache_manager.clone(),
- group_type_cache: state.group_type_cache_manager.clone(),
- person_property_overrides,
- group_property_overrides,
- groups,
- hash_key_override,
- flag_keys,
- optimize_experience_continuity_lookups: state
- .config
- .optimize_experience_continuity_lookups
- .0,
- parallel_eval_threshold: state.config.parallel_eval_threshold,
- rayon_dispatcher: state.rayon_dispatcher.clone(),
- skip_writes: detailed_analysis.unwrap_or(false)
- || only_use_override_person_properties.unwrap_or(false)
- || *state.config.skip_writes,
- cohort_membership_provider: state.cohort_membership_provider.clone(),
- enable_realtime_cohort_evaluation: state
+ // No flag is left, or every flag is in the filter set (inactive, deleted, runtime/tag
+ // mismatch), so there is nothing to evaluate. This is O(1) since the filter set includes
+ // deleted flags.
+ let mut response = if filtered_flags.filtered_out_flag_ids.len() >= filtered_flags.flags.len() {
+ FlagsResponse::new(false, HashMap::new(), None, request_id)
+ } else {
+ let persons_db_deadline = state
.config
- .realtime_cohort_evaluation_team_ids
- .includes_team(team_id),
- use_explicit_exact_matching,
- membership_stamp_policy: state.config.realtime_cohort_membership_stamp_policy,
- detailed_analysis: detailed_analysis.unwrap_or(false),
- only_use_override_person_properties: only_use_override_person_properties.unwrap_or(false),
- persons_db_deadline: persons_db_deadline.map(|deadline| deadline.at),
+ .persons_db_deadline(tokio::time::Instant::now(), Some(request_entered_at));
+ with_canonical_log(|log| {
+ log.persons_db_deadline_bound = persons_db_deadline.map(|deadline| deadline.bound)
+ });
+
+ let ctx = FeatureFlagEvaluationContext {
+ team_id,
+ team_timezone,
+ distinct_id,
+ device_id,
+ feature_flags: filtered_flags,
+ persons_reader: state.database_pools.persons_reader.clone(),
+ persons_writer: state.database_pools.persons_writer.clone(),
+ non_persons_reader: state.database_pools.non_persons_reader.clone(),
+ non_persons_writer: state.database_pools.non_persons_writer.clone(),
+ cohort_cache: state.cohort_cache_manager.clone(),
+ group_type_cache: state.group_type_cache_manager.clone(),
+ person_property_overrides,
+ group_property_overrides,
+ groups,
+ hash_key_override,
+ flag_keys,
+ optimize_experience_continuity_lookups: state
+ .config
+ .optimize_experience_continuity_lookups
+ .0,
+ parallel_eval_threshold: state.config.parallel_eval_threshold,
+ rayon_dispatcher: state.rayon_dispatcher.clone(),
+ skip_writes: detailed_analysis.unwrap_or(false)
+ || only_use_override_person_properties.unwrap_or(false)
+ || *state.config.skip_writes,
+ cohort_membership_provider: state.cohort_membership_provider.clone(),
+ enable_realtime_cohort_evaluation: state
+ .config
+ .realtime_cohort_evaluation_team_ids
+ .includes_team(team_id),
+ use_explicit_exact_matching,
+ membership_stamp_policy: state.config.realtime_cohort_membership_stamp_policy,
+ detailed_analysis: detailed_analysis.unwrap_or(false),
+ only_use_override_person_properties: only_use_override_person_properties
+ .unwrap_or(false),
+ persons_db_deadline: persons_db_deadline.map(|deadline| deadline.at),
+ };
+
+ evaluation::evaluate_feature_flags(ctx, request_id).await?
};
- evaluation::evaluate_feature_flags(ctx, request_id).await
+ // A response for a team whose cached payload had a flag skipped reports an error, so that
+ // an SDK merges it into its stored flags and keeps the skipped flag's last value.
+ response.errors_while_computing_flags |= has_unparseable_flags;
+ Ok(response)
}
#[cfg(test)]
diff --git a/rust/feature-flags/src/handler/tests.rs b/rust/feature-flags/src/handler/tests.rs
index 21b903e277e1..379c874ee568 100644
--- a/rust/feature-flags/src/handler/tests.rs
+++ b/rust/feature-flags/src/handler/tests.rs
@@ -1464,6 +1464,7 @@ async fn test_fetch_and_filter_preserves_evaluation_metadata() {
dependency_stages: vec![vec![1], vec![2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(2, std::collections::HashSet::from([1]))]),
+ ..Default::default()
};
let wrapper = HypercacheFlagsWrapper {
flags: flags.clone(),
diff --git a/rust/feature-flags/src/metrics/consts.rs b/rust/feature-flags/src/metrics/consts.rs
index 643110201dc0..ffa69370793c 100644
--- a/rust/feature-flags/src/metrics/consts.rs
+++ b/rust/feature-flags/src/metrics/consts.rs
@@ -441,13 +441,15 @@ pub const FLAG_CONDITION_SKIPPED_COUNTER: &str = "flags_condition_skipped_total"
// V2 ingress outcomes use fixed labels and never include configuration values.
pub const FLAG_V2_PARSE_COUNTER: &str = "flags_v2_config_parse_total";
-// Incremented once per flag left out of a team's payload because its `filters` JSON
-// does not deserialize into FlagFilters. A property filter with no `"type"` key is
-// one such blob, because PropertyFilter requires prop_type, and serde fails the
-// whole outer struct. This counts every dropped flag, which is a superset of the
-// flags the two builders disagree about: Python keeps an active or referenced flag
-// that this drops, but an inactive, unreferenced flag is dropped by both builders.
-// Team id and flag key are in the companion warn log, not in metric labels (cardinality).
+// Incremented once per flag left out of a team's payload because its JSON does not
+// deserialize: the `filters` column on the PG read, or the whole flag object on the
+// hypercache read. A property filter with no `"type"` key is one such blob, because
+// PropertyFilter requires prop_type, and serde fails the whole outer struct. This
+// counts every dropped flag, which is a superset of the flags the two builders
+// disagree about: Python keeps an active or referenced flag that this drops, but an
+// inactive, unreferenced flag is dropped by both builders. The `operation` label on
+// the companion tombstone counter says which read path dropped it. Team id and flag
+// key are in the companion warn log, not in metric labels (cardinality).
pub const FLAG_MALFORMED_FILTER_COUNTER: &str = "flags_flag_malformed_filter_total";
// Incremented once per team read that left out at least one flag for the reason
// above. FLAG_MALFORMED_FILTER_COUNTER divided by this gives the mean flags
diff --git a/rust/feature-flags/src/utils/test_graph_utils.rs b/rust/feature-flags/src/utils/test_graph_utils.rs
index e8a00379b62c..80433f59faac 100644
--- a/rust/feature-flags/src/utils/test_graph_utils.rs
+++ b/rust/feature-flags/src/utils/test_graph_utils.rs
@@ -1023,6 +1023,7 @@ mod evaluation_metadata_serde_tests {
(3, HashSet::new()),
(4, HashSet::new()),
]),
+ ..Default::default()
};
let json = serde_json::to_string(&original).expect("serialize");
@@ -1043,6 +1044,7 @@ mod evaluation_metadata_serde_tests {
dependency_stages: vec![vec![1]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::new(),
+ ..Default::default()
};
let json = serde_json::to_string(&original).expect("serialize");
@@ -1224,6 +1226,7 @@ mod precomputed_dependency_graph_tests {
dependency_stages: vec![vec![3]],
flags_with_missing_deps: vec![1],
transitive_deps: HashMap::from([(1, HashSet::from([2])), (3, HashSet::new())]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1249,6 +1252,7 @@ mod precomputed_dependency_graph_tests {
dependency_stages: vec![vec![2], vec![1]],
flags_with_missing_deps: vec![1, 2],
transitive_deps: HashMap::from([(1, HashSet::from([2])), (2, HashSet::new())]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1303,6 +1307,7 @@ mod precomputed_dependency_graph_tests {
dependency_stages: vec![vec![2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(2, HashSet::new())]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -1401,6 +1406,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::from([3])),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1444,6 +1450,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::new()),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1473,6 +1480,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::from([1])),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1523,6 +1531,7 @@ mod precomputed_dependency_graph_tests {
dependency_stages: vec![vec![1, 2]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new()), (2, HashSet::new())]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -1553,6 +1562,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::new()),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
cohorts: None,
};
@@ -1597,6 +1607,7 @@ mod precomputed_dependency_graph_tests {
(3, HashSet::new()),
(4, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1637,6 +1648,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::new()),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1657,6 +1669,7 @@ mod precomputed_dependency_graph_tests {
dependency_stages: vec![vec![1]],
flags_with_missing_deps: vec![],
transitive_deps: HashMap::from([(1, HashSet::new())]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1688,6 +1701,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::new()),
(3, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1723,6 +1737,7 @@ mod precomputed_dependency_graph_tests {
(3, HashSet::new()),
(4, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
@@ -1801,6 +1816,7 @@ mod precomputed_dependency_graph_tests {
(2, HashSet::from([1])),
(999, HashSet::new()),
]),
+ ..Default::default()
}),
..Default::default()
};
diff --git a/rust/feature-flags/src/utils/test_utils.rs b/rust/feature-flags/src/utils/test_utils.rs
index 7eded8e114da..517502696784 100644
--- a/rust/feature-flags/src/utils/test_utils.rs
+++ b/rust/feature-flags/src/utils/test_utils.rs
@@ -106,6 +106,22 @@ pub async fn insert_new_team_in_redis(
Ok(team)
}
+/// A cached flag that the serving read cannot deserialize. Its property filter has no
+/// `"type"` key, and `PropertyFilter` requires one.
+pub fn unparseable_flag_json(id: i32, key: &str, team_id: i32) -> Value {
+ json!({
+ "id": id,
+ "key": key,
+ "team_id": team_id,
+ "active": true,
+ "deleted": false,
+ "filters": {"groups": [{
+ "properties": [{"key": "email", "value": "[email protected]"}],
+ "rollout_percentage": 100
+ }]}
+ })
+}
+
/// Write flags to the hypercache key for `team_id` in Redis.
/// Auto-generates single-stage `evaluation_metadata` with no transitive-dependency
/// information (all flags in one stage, `transitive_deps[id] = []` for each flag).
@@ -1335,6 +1351,7 @@ pub fn flag_list_with_metadata_and_filter(
dependency_stages: stages,
flags_with_missing_deps,
transitive_deps,
+ ..Default::default()
});
FeatureFlagList {
diff --git a/rust/feature-flags/tests/test_flags.rs b/rust/feature-flags/tests/test_flags.rs
index b59560afa9aa..b8a5007fe208 100644
--- a/rust/feature-flags/tests/test_flags.rs
+++ b/rust/feature-flags/tests/test_flags.rs
@@ -17,7 +17,7 @@ use feature_flags::config::DEFAULT_TEST_CONFIG;
use feature_flags::utils::test_utils::{
insert_config_in_hypercache, insert_flags_for_team_in_redis,
insert_flags_with_metadata_for_team_in_redis, insert_new_team_in_redis, setup_pg_reader_client,
- setup_redis_client, update_team_in_hypercache, TestContext,
+ setup_redis_client, unparseable_flag_json, update_team_in_hypercache, TestContext,
};
pub mod common;
@@ -6405,3 +6405,46 @@ async fn it_degrades_person_flags_when_the_persons_db_stops_answering() -> Resul
Ok(())
}
+
+#[rstest]
+#[case::no_flag_left(None, &[])]
+#[case::only_an_inactive_flag_left(Some(false), &[])]
+#[case::a_healthy_flag_left(Some(true), &["other-flag"])]
+#[tokio::test]
+async fn it_reports_errors_when_the_serving_read_skips_a_flag(
+ #[case] other_flag_active: Option<bool>,
+ #[case] expected_keys: &[&str],
+) -> Result<()> {
+ let config = DEFAULT_TEST_CONFIG.clone();
+
+ let client = setup_redis_client(Some(config.redis_url.clone())).await;
+ let team = insert_new_team_in_redis(client.clone()).await.unwrap();
+ let context = TestContext::new(None).await;
+ context.insert_new_team(Some(team.id)).await.unwrap();
+
+ let mut flags = vec![unparseable_flag_json(1, "malformed-flag", team.id)];
+ if let Some(active) = other_flag_active {
+ flags.push(json!({
+ "id": 2,
+ "key": "other-flag",
+ "active": active,
+ "deleted": false,
+ "team_id": team.id,
+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]}
+ }));
+ }
+ insert_flags_for_team_in_redis(client, team.id, Some(Value::Array(flags).to_string())).await?;
+
+ let server = ServerHandle::for_config(config).await;
+ let payload = json!({"token": team.api_token, "distinct_id": "user_distinct_id"}).to_string();
+ let res = server.send_flags_request(payload, Some("2"), None).await;
+
+ assert_eq!(res.status(), StatusCode::OK);
+ let body = res.json::<FlagsResponse>().await?;
+ assert!(body.errors_while_computing_flags);
+ let mut keys: Vec<&str> = body.flags.keys().map(String::as_str).collect();
+ keys.sort_unstable();
+ assert_eq!(keys, expected_keys);
+
+ Ok(())
+}