:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""
Flags HyperCache for feature-flags service.
This module provides a HyperCache that stores feature flags for the feature-flags service.
Unlike the local_evaluation.py cache which provides rich data for SDKs (including cohorts
and group type mappings), this cache provides flag data plus the cohort definitions that
those flags actually reference.
The cache is automatically invalidated when:
- FeatureFlag models are created, updated, or deleted
- Team models are created or deleted (to ensure flag caches are cleaned up)
- EvaluationContext / FeatureFlagEvaluationContext models are created or deleted
- Cohort definitions are created, updated, or deleted (not recalculation-only saves)
- Hourly refresh job detects expiring entries (TTL < 24h)
Cache Key Pattern:
- Uses team_id as the key
- Stored in both Redis and S3 via HyperCache
Configuration:
- Redis TTL: 7 days (configurable via FLAGS_CACHE_TTL env var)
- Miss TTL: 1 day (configurable via FLAGS_CACHE_MISS_TTL env var)
Manual operations:
from products.feature_flags.backend.flags_cache import clear_flags_cache
clear_flags_cache(team_id)
"""
import sys
from collections import defaultdict, deque
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import UTC, datetime, timedelta
from functools import partial
from typing import TYPE_CHECKING, Any, Literal
if TYPE_CHECKING:
from django.db.models import QuerySet
from django.conf import settings
from django.contrib.postgres.aggregates import ArrayAgg
from django.db import transaction
from django.db.models import Exists, OuterRef, Q
from django.db.models.signals import post_delete, post_save
from django.dispatch import receiver
from django.utils import timezone
import structlog
import posthoganalytics
from celery.exceptions import SoftTimeLimitExceeded
from posthog.caching.flags_redis_cache import FLAGS_DEDICATED_CACHE_ALIAS
from posthog.kafka_client.routing import producer_scope
from posthog.kafka_client.topics import KAFKA_FLAGS_CACHE_INVALIDATION
from posthog.metrics import TOMBSTONE_COUNTER
from posthog.models.team import Team
from posthog.ph_client import feature_enabled_or_false
from posthog.storage.cache_expiry_manager import (
CacheRefreshCounts,
RefreshPacing,
cleanup_stale_expiry_tracking as cleanup_generic,
get_teams_with_expiring_caches,
refresh_expiring_caches,
)
from posthog.storage.hypercache import HyperCache
from posthog.storage.hypercache_manager import (
HyperCacheManagementConfig,
get_cache_stats as get_cache_stats_generic,
)
from products.cohorts.backend.models.cohort import Cohort
from products.cohorts.backend.models.dependencies import extract_cohort_dependencies
from products.experiments.backend.models.experiment import Experiment, live_experiment_exists
from products.feature_flags.backend.facade.config import (
ConfigFormatError,
UnsupportedConfig,
decode_config,
detect_config_format,
)
from products.feature_flags.backend.facade.references import FlagReferences, references
from products.feature_flags.backend.flags_cache_messages import FlagsCacheInvalidation
from products.feature_flags.backend.models.evaluation_context import FeatureFlagEvaluationContext
from products.feature_flags.backend.models.feature_flag import FeatureFlag, get_feature_flags, serialize_feature_flags
logger = structlog.get_logger(__name__)
# Sorted set key for tracking cache expirations
FLAGS_CACHE_EXPIRY_SORTED_SET = "flags_cache_expiry"
def _is_unevaluable(flag_data: dict[str, Any]) -> bool:
"""Whether ``active``/``deleted`` make the matcher skip this serialized flag before
it reads the filters.
The matcher filters out more flags per request (survey, evaluation runtime,
evaluation contexts), but those are request-scoped and must not gate what gets
written to a cache shared by every request.
Inverse of ``is_evaluable`` in rust/feature-flags/src/flags/cache_builder.rs. A
flag missing ``active`` counts as evaluable so a serializer regression leaves the
targeting in the payload instead of overwriting it. That protects the bytes only:
Rust's ``FeatureFlag.active`` is ``#[serde(default)]``, so a payload without the
field reads as inactive there and is filtered out regardless.
"""
return not flag_data.get("active", True) or flag_data.get("deleted", False)
def _evaluated_references(flag_data: dict[str, Any]) -> FlagReferences:
"""The references of a serialized flag the matcher reads, and none for an unevaluable
flag, whose filters it skips.
Only flags that passed ``_omit_unsupported_flags`` are serialized, so the document is
a readable config version 1 or a supported v2 document.
"""
if _is_unevaluable(flag_data):
return FlagReferences()
config = decode_config(flag_data.get("filters", {}))
return FlagReferences() if isinstance(config, UnsupportedConfig) else references(config)
def _extract_direct_dependency_ids(flag_data: dict[str, Any]) -> set[int]:
"""
Extract direct flag dependency IDs from a serialized flag's filters.
Inactive/deleted flags return empty deps before their filters are read, to
match Rust's extract_dependencies behavior. A ``key`` that is not an integer id is skipped.
"""
return set(_evaluated_references(flag_data).flag_ids)
def _validates_v2(filters: Mapping[str, Any]) -> bool:
"""Whether the shared validator admits a v2 document under the deployed filter-size
limit, the bound the Rust reader also applies. Rust may still reject what it cannot
read; the service then returns that one flag with ``failed: true`` and a
``flag_data_parsing_error`` reason and still evaluates the rest.
"""
# Deferred: the validator imports posthog.hogql, which must stay off the django.setup() path.
from products.feature_flags.backend.facade.config_validation import ( # noqa: PLC0415
ConfigValidationError,
ValidationLimits,
validate_config,
)
limits = ValidationLimits(
max_config_bytes=settings.MAX_FEATURE_FLAG_FILTER_SIZE_BYTES, max_metadata_bytes=sys.maxsize
)
try:
validate_config(filters, limits=limits)
except ConfigValidationError:
return False
return True
def _stored_dependency_ids(flag: FeatureFlag) -> set[int] | None:
"""The flag ids a stored row's release conditions reference, or ``None`` when this
cache cannot carry the row.
A non-object document and an unsupported discriminator are rejected whatever the
row's lifecycle. A v2 document is carried verbatim only when the row is active and
``_validates_v2`` admits it. Any other v2 row is rejected, so an inactive v2 row is
never blanked into a v1-shaped entry. An unevaluable v1 object is not read, since
``_blank_inactive_filters`` empties it; an evaluable one whose conditions cannot be
read is rejected instead of failing the team.
"""
filters = flag.filters
if not isinstance(filters, Mapping):
return None
kind = detect_config_format(filters).kind
if kind == "v2":
if not (flag.active and not flag.deleted and _validates_v2(filters)):
return None
elif kind != "v1":
return None
elif not flag.active or flag.deleted:
return set()
try:
return set(references(decode_config(filters)).flag_ids)
except (AttributeError, TypeError, ConfigFormatError):
return None
def _omit_unsupported_flags(team_id: int, flags: list[FeatureFlag]) -> list[FeatureFlag]:
"""Drop the stored rows this cache cannot carry, and the rows whose dependency
conditions reference one, transitively.
A dependent goes with its target because the matcher pre-seeds every kept
unevaluable id as false: an unsupported target left in that bucket would satisfy a
``flag_evaluates_to: false`` condition it never evaluated, and one dropped without
its dependents would leave them resolving a missing dependency. Ordinary inactive v1
targets keep their established kept-and-seeded-false behavior. Stored rows are not
modified; ``build_flags_cache`` in rust/feature-flags/src/flags/cache_builder.rs
applies the same rules, so both writers publish the same flag set.
"""
excluded: set[int] = set()
dependents: dict[int, set[int]] = defaultdict(set)
for flag in flags:
dependency_ids = _stored_dependency_ids(flag)
if dependency_ids is None:
excluded.add(flag.id)
continue
for dependency_id in dependency_ids:
dependents[dependency_id].add(flag.id)
unsupported = sorted(excluded)
queue = deque(excluded)
while queue:
for dependent_id in dependents.get(queue.popleft(), ()):
if dependent_id not in excluded:
excluded.add(dependent_id)
queue.append(dependent_id)
if excluded:
logger.warning(
"Omitted flags the service cache cannot carry",
team_id=team_id,
unsupported_flag_ids=unsupported,
dependent_flag_ids=sorted(excluded.difference(unsupported)),
)
return [flag for flag in flags if flag.id not in excluded]
# Cohort model fields that change only during recalculation, not definition edits.
# Used by the post_save signal to avoid rebuilding the flags cache on every
# static cohort recalculation or count update.
# NOTE: cohort_type is included because calculate_people_ch() always saves it in
# update_fields even when unchanged. The rare actual cohort_type change (realtime
# exceeding person limit) uses Cohort.objects.filter().update() which bypasses signals.
_COHORT_RECALCULATION_FIELDS = frozenset(
[
"count",
"version",
"pending_version",
"is_calculating",
"last_calculation",
"last_calculation_duration_ms",
"errors_calculating",
"last_error_at",
"last_import_total_count",
"last_import_unmatched_count",
# NOTE: `groups` is the legacy cohort-condition field (deprecated in favour of
# `filters`). calculate_people_ch() always saves it in update_fields even when
# unchanged (see cohort.py:347). Real definition changes go through a full save
# (update_fields=None), so they still trigger invalidation.
"groups",
"cohort_type",
]
)
def _extract_cohort_ids_from_flag_filters(flags_data: list[dict[str, Any]]) -> set[int]:
"""Extract cohort IDs directly referenced in active flag filters.
Inactive/deleted flags are skipped before their filters are read; every other
flag here passed ``_omit_unsupported_flags``.
"""
cohort_ids: set[int] = set()
for flag in flags_data:
cohort_ids.update(_evaluated_references(flag).cohort_ids)
return cohort_ids
_MAX_COHORT_DEPENDENCY_DEPTH = 20
def _load_cohorts_with_deps(seed_ids: set[int], **team_filter: Any) -> dict[int, Cohort]:
"""BFS-load cohorts by seed IDs, resolving transitive cohort-on-cohort deps.
Args:
seed_ids: Initial cohort IDs to load.
**team_filter: Passed to Cohort.objects.filter() for team scoping,
e.g. team_id=5 or team_id__in={5, 6}.
Returns:
Dict mapping cohort PK to loaded Cohort instance.
"""
if not seed_ids:
return {}
all_ids = set(seed_ids)
ids_to_load = set(seed_ids)
loaded: dict[int, Cohort] = {}
depth = 0
while ids_to_load:
if depth >= _MAX_COHORT_DEPENDENCY_DEPTH:
logger.warning(
"Cohort dependency depth limit reached",
depth=depth,
remaining_ids=ids_to_load,
)
break
depth += 1
newly_loaded: list[Cohort] = []
# nosemgrep: idor-lookup-without-team — team scope is enforced via **team_filter kwargs (team_id or team__project_id__in passed by caller).
for cohort in Cohort.objects.filter(pk__in=ids_to_load, deleted=False, **team_filter):
loaded[cohort.pk] = cohort
newly_loaded.append(cohort)
ids_to_load_next: set[int] = set()
for cohort in newly_loaded:
for dep_id in extract_cohort_dependencies(cohort):
if dep_id not in all_ids:
all_ids.add(dep_id)
ids_to_load_next.add(dep_id)
ids_to_load = ids_to_load_next
return loaded
def _get_referenced_cohorts(team_id: int, flags_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Fetch cohort definitions referenced by flags, including transitive cohort-on-cohort deps."""
direct_ids = _extract_cohort_ids_from_flag_filters(flags_data)
loaded = _load_cohorts_with_deps(direct_ids, team_id=team_id)
return [_serialize_cohort(c) for c in loaded.values()]
def _serialize_cohort(cohort: Cohort) -> dict[str, Any]:
"""Serialize a Cohort to a dict matching the Rust Cohort struct field names.
HYPERCACHE CONTRACT: These field names must match the Rust Cohort struct in
rust/feature-flags/src/cohorts/cohort_models.rs. Field changes must follow
the expand-and-contract pattern. The contract test will catch mismatches:
pytest products/feature_flags/backend/test/test_flags_cache.py -k "test_serializer_output_matches_fixture_schema"
Note: deleted, is_calculating, is_static, errors_calculating, and groups
are required by the Rust struct (no #[serde(default)]), so omitting them
causes a deserialization failure.
"""
return {
"id": cohort.id,
"name": cohort.name,
"description": cohort.description,
"team_id": cohort.team_id,
"deleted": cohort.deleted,
"filters": cohort.filters,
"query": cohort.query,
"version": cohort.version,
"pending_version": cohort.pending_version,
"count": cohort.count,
"is_calculating": cohort.is_calculating,
"is_static": cohort.is_static,
"errors_calculating": cohort.errors_calculating,
"groups": cohort.groups,
"created_by_id": cohort.created_by_id,
"cohort_type": cohort.cohort_type,
"condition_type": cohort.condition_type,
"last_backfill_person_properties_at": (
cohort.last_backfill_person_properties_at.isoformat() if cohort.last_backfill_person_properties_at else None
),
"last_backfill_events_at": (
cohort.last_backfill_events_at.isoformat() if cohort.last_backfill_events_at else None
),
"last_realtime_cohort_calculation_at": (
cohort.last_realtime_cohort_calculation_at.isoformat()
if cohort.last_realtime_cohort_calculation_at
else None
),
}
def _compute_flag_dependencies(flags_data: list[dict[str, Any]]) -> dict[str, Any]:
"""
Compute flag dependency metadata and return evaluation metadata.
Returns a dict with:
- dependency_stages: list of lists of flag IDs grouped by evaluation stage,
stage 0 (no deps) first. Flags in the same stage can be evaluated in parallel.
- flags_with_missing_deps: sorted list of flag IDs with missing, cyclic, or
transitively broken dependencies.
- transitive_deps: dict mapping stringified flag ID to sorted list of all
transitive dependency flag IDs.
Uses Kahn's algorithm (layered topological sort) to match the Rust fallback
path's petgraph-based cycle handling: all cycle participants are excluded
from stages, not just back-edge targets.
"""
id_to_flag: dict[int, dict[str, Any]] = {}
for flag in flags_data:
flag_id = flag["id"]
id_to_flag[flag_id] = flag
# Build direct dependency edges (may reference unknown flag IDs)
direct_deps: dict[int, set[int]] = {}
for flag_id in id_to_flag:
direct_deps[flag_id] = _extract_direct_dependency_ids(id_to_flag[flag_id])
# Track flags with missing dependencies (dep ID not in id_to_flag)
has_missing: dict[int, bool] = {
fid: any(dep_id not in id_to_flag for dep_id in deps) for fid, deps in direct_deps.items()
}
# Build in-degree map (only count edges to known flags)
in_degree: dict[int, int] = dict.fromkeys(id_to_flag, 0)
# reverse_deps: flag_id → set of flags that depend on it
reverse_deps: dict[int, set[int]] = {fid: set() for fid in id_to_flag}
for flag_id, deps in direct_deps.items():
for dep_id in deps:
if dep_id in id_to_flag:
in_degree[flag_id] += 1
reverse_deps[dep_id].add(flag_id)
# Kahn's algorithm: peel layers of zero-in-degree nodes
dependency_stages: list[list[int]] = []
transitive_deps: dict[int, set[int]] = {}
queue = sorted(fid for fid, deg in in_degree.items() if deg == 0)
while queue:
for fid in queue:
# Compute transitive deps: union of each dep's transitive closure + direct deps
td: set[int] = set()
for dep_id in direct_deps[fid]:
if dep_id in id_to_flag:
td.add(dep_id)
td.update(transitive_deps[dep_id])
if has_missing[dep_id]:
has_missing[fid] = True
transitive_deps[fid] = td
dependency_stages.append(queue)
next_queue: list[int] = []
for fid in queue:
for dependent_id in reverse_deps[fid]:
in_degree[dependent_id] -= 1
if in_degree[dependent_id] == 0:
next_queue.append(dependent_id)
queue = sorted(next_queue)
# Flags still with in_degree > 0 are in cycles.
cycled_flags = {fid for fid, deg in in_degree.items() if deg > 0}
for fid in cycled_flags:
has_missing[fid] = True
return {
"dependency_stages": dependency_stages,
"flags_with_missing_deps": sorted(fid for fid, m in has_missing.items() if m),
"transitive_deps": {str(fid): sorted(transitive_deps.get(fid, set())) for fid in id_to_flag},
}
def _drop_unreferenced_unevaluable_flags(flags_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Return the flags worth caching: evaluable ones, plus unevaluable ones that
another flag's dependency conditions reference.
An unreferenced inactive/deleted flag is dead weight — the matcher filters it
out before reading it and nothing resolves against it. A referenced one is
load-bearing: the matcher pre-seeds its id as false, so a dependent with
``flag_evaluates_to: false`` on a disabled flag still matches; dropping it
would turn that dependent into a missing-dependency miss.
``_extract_direct_dependency_ids`` returns nothing for unevaluable flags, so
only evaluable flags contribute references: a chain of inactive flags keeps
just the hop an evaluable flag points at.
"""
referenced_ids: set[int] = set()
for flag in flags_data:
referenced_ids |= _extract_direct_dependency_ids(flag)
return [flag for flag in flags_data if not _is_unevaluable(flag) or flag["id"] in referenced_ids]
def _blank_inactive_filters(flags_data: list[dict[str, Any]]) -> None:
"""Empty the ``filters`` of flags that can never be evaluated, in place.
The matcher skips these flags before it reads filters, so the blob is
unreachable weight in Redis, in S3, and in the service's in-memory cache. The
flag entry itself stays, so a dependency condition on it still resolves to
false rather than raising DependencyNotFound.
Order-independent: dependency and cohort extraction guard on ``_is_unevaluable``
themselves.
"""
for flag in flags_data:
if _is_unevaluable(flag):
# Matches an empty Rust ``FlagFilters``, whose ``groups`` has no
# ``skip_serializing_if`` and so serializes as a key rather than ``{}``.
# Both writers of this entry emit this same blank shape.
flag["filters"] = {"groups": []}
def _build_flags_payload(
flags_data: list[dict[str, Any]],
cohorts: list[dict[str, Any]],
) -> dict[str, Any]:
"""Assemble the ``flags.json`` wrapper: drop unreferenced unevaluable flags,
compute dependency metadata on the survivors, and blank the kept unevaluable
flags' filters.
These steps live here so no writer can assemble a payload without them, and so
the metadata can never be computed on a pre-drop flag list. The Rust side
reaches the same point through a single ``build_flags_cache``.
"""
flags_data = _drop_unreferenced_unevaluable_flags(flags_data)
evaluation_metadata = _compute_flag_dependencies(flags_data)
_blank_inactive_filters(flags_data)
return {"flags": flags_data, "evaluation_metadata": evaluation_metadata, "cohorts": cohorts}
def _get_feature_flags_for_service(team: Team) -> dict[str, Any]:
"""
Get feature flags for the feature-flags service.
HYPERCACHE CONTRACT: The top-level keys (flags, evaluation_metadata, cohorts)
must match rust/feature-flags/src/flags/flag_models.rs::HypercacheFlagsWrapper.
Changes to this structure must follow the expand-and-contract pattern.
Fetches the team's evaluable feature flags (active, not deleted), plus the
inactive flags another flag's dependency conditions reference, wrapped in a
dict that HyperCache can serialize. The actual flag data is in the "flags"
key as a list of flag dictionaries. Rows stored in a config format this cache
cannot carry, and their dependents, are omitted (``_omit_unsupported_flags``).
Encrypted remote config flags are excluded since they can only be accessed via
the dedicated /remote_config endpoint which handles decryption. Including them
in /flags would return unusable encrypted ciphertext.
Returns:
dict: {"flags": [...], "evaluation_metadata": {...}, "cohorts": [...]} where
flags is a list of flag dictionaries, evaluation_metadata contains pre-computed
dependency metadata (stages, missing deps, transitive deps), and cohorts contains
serialized cohort definitions referenced by the flags (including transitive deps).
"""
flags = _omit_unsupported_flags(team.id, get_feature_flags(team=team, exclude_encrypted_payloads=True))
flags_data = serialize_feature_flags(flags)
cohorts = _get_referenced_cohorts(team.id, flags_data)
payload = _build_flags_payload(flags_data, cohorts)
logger.info(
"Loaded feature flags for service cache",
team_id=team.id,
project_id=team.project_id,
flag_count=len(payload["flags"]),
cohort_count=len(cohorts),
)
return payload
def _get_feature_flags_for_teams_batch(teams: list[Team]) -> dict[int, dict[str, Any]]:
"""
Batch load feature flags for multiple teams in one query.
This avoids N+1 queries by loading all flags for all teams at once,
then grouping them by team_id.
Encrypted remote config flags are excluded since they can only be accessed via
the dedicated /remote_config endpoint which handles decryption. Including them
in /flags would return unusable encrypted ciphertext.
Args:
teams: List of Team objects to load flags for
Returns:
Dict mapping team_id to {"flags": [...], "evaluation_metadata": {...}, "cohorts": [...]} for each team
"""
if not teams:
return {}
# Load all flags for all teams in one query with evaluation tags pre-loaded.
# Disabled flags (active=False) are loaded too: ones referenced by another
# flag's dependencies stay in the payload and evaluate as false, and
# _build_flags_payload drops the rest.
# Exclude encrypted payload flags - they can only be accessed via the
# dedicated /remote_config endpoint which handles decryption.
# Note: We intentionally don't select_related("team") here because we only need
# team_id (already on the model) for grouping, and the Team objects are already
# loaded by the caller. Avoiding the join saves memory.
all_flags = list(
FeatureFlag.objects.filter(team__in=teams)
.exclude(has_encrypted_payloads=True)
.annotate(
evaluation_tag_names_agg=ArrayAgg(
"flag_evaluation_contexts__evaluation_context__name",
filter=Q(flag_evaluation_contexts__isnull=False),
distinct=True,
),
has_experiment_agg=live_experiment_exists(),
)
)
# Transfer aggregated tag names to model instances
for flag in all_flags:
flag._evaluation_tag_names = flag.evaluation_tag_names_agg or []
flag._has_experiment = flag.has_experiment_agg
# Group flags by team_id
flags_by_team_id: dict[int, list[FeatureFlag]] = defaultdict(list)
for flag in all_flags:
flags_by_team_id[flag.team_id].append(flag)
# Serialize flags for each team and collect all referenced cohort IDs
flags_data_by_team: dict[int, list[dict[str, Any]]] = {}
all_cohort_ids: set[int] = set()
for team in teams:
team_flags = _omit_unsupported_flags(team.id, flags_by_team_id.get(team.id, []))
flags_data = serialize_feature_flags(team_flags)
flags_data_by_team[team.id] = flags_data
all_cohort_ids.update(_extract_cohort_ids_from_flag_filters(flags_data))
# Batch-load all referenced cohorts across all teams (including transitive deps)
team_ids = {t.id for t in teams}
loaded_cohorts = _load_cohorts_with_deps(all_cohort_ids, team_id__in=team_ids)
# Group loaded cohorts by team_id
cohorts_by_team: dict[int, list[dict[str, Any]]] = defaultdict(list)
for cohort in loaded_cohorts.values():
cohorts_by_team[cohort.team_id].append(_serialize_cohort(cohort))
# Build result for each team
result: dict[int, dict[str, Any]] = {}
for team in teams:
team_cohorts = cohorts_by_team.get(team.id, [])
payload = _build_flags_payload(flags_data_by_team[team.id], team_cohorts)
logger.info(
"Loaded feature flags for service cache (batch)",
team_id=team.id,
project_id=team.project_id,
flag_count=len(payload["flags"]),
cohort_count=len(team_cohorts),
)
result[team.id] = payload
return result
# HyperCache instance for feature-flags service
# Use dedicated flags cache alias if available, otherwise defaults to default cache
flags_hypercache = HyperCache(
namespace="feature_flags",
value="flags.json",
load_fn=lambda key: _get_feature_flags_for_service(HyperCache.team_from_key(key)),
cache_ttl=settings.FLAGS_CACHE_TTL,
cache_miss_ttl=settings.FLAGS_CACHE_MISS_TTL,
cache_alias=FLAGS_DEDICATED_CACHE_ALIAS if FLAGS_DEDICATED_CACHE_ALIAS in settings.CACHES else None,
batch_load_fn=_get_feature_flags_for_teams_batch,
expiry_sorted_set_key=FLAGS_CACHE_EXPIRY_SORTED_SET,
# Etag is consumed by the Rust feature-flags service in-memory cache
# (FlagDefinitionsCache), keyed on (team_id, etag). Without this, the cache
# bypass branch fires on every request and the perf opt is wasted.
enable_etag=True,
)
def get_flags_from_cache(team: Team) -> list[dict[str, Any]] | None:
"""
Get feature flags from the cache for a team.
Only operates when FLAGS_REDIS_URL is configured to avoid reading from/writing to shared cache.
Args:
team: The team to get flags for
Returns:
list: Flag dictionaries (empty list if team has zero flags)
None: Cache miss or FLAGS_REDIS_URL not configured
"""
if not settings.FLAGS_REDIS_URL:
return None
result = flags_hypercache.get_from_cache(team)
if result is None:
return None
return result.get("flags", [])
def update_flags_cache(team: Team | int, ttl: int | None = None) -> bool:
"""
Update the flags cache for a team.
This explicitly updates both Redis and S3 with the latest flag data.
Only operates when FLAGS_REDIS_URL is configured to avoid writing to shared cache.
Expiry tracking is handled automatically by HyperCache.set_cache_value().
Args:
team: Team object or team ID
ttl: Optional custom TTL in seconds (defaults to FLAGS_CACHE_TTL)
Returns:
True if cache update succeeded, False otherwise
"""
if not settings.FLAGS_REDIS_URL:
return False
success = flags_hypercache.update_cache(team, ttl=ttl)
if not success:
team_id = team.id if isinstance(team, Team) else team
logger.warning("Failed to update flags cache", team_id=team_id)
return success
def verify_team_flags(
team: Team,
db_batch_data: dict | None = None,
cache_batch_data: dict | None = None,
verbose: bool = False,
) -> dict:
"""
Verify a team's flags cache against the database.
Args:
team: Team to verify
db_batch_data: Pre-loaded DB data from batch_load_fn (keyed by team.id)
cache_batch_data: Pre-loaded cache data from batch_get_from_cache (keyed by team.id)
verbose: If True, include detailed diffs with flag keys and field-level differences
Returns:
Dict with 'status' ("match", "miss", "mismatch") and 'issue' type.
When verbose=True, includes 'diffs' list with detailed diff information.
"""
# Get cached data - use pre-loaded batch data if available (single MGET for whole batch).
# The etag rides on the same MGET when enable_etag=True, so the MISSING_ETAG check
# below stays per-chunk and never re-introduces a per-team Redis GET.
if cache_batch_data and team.id in cache_batch_data:
cached_data, source, cached_etag = cache_batch_data[team.id]
else:
# Single-team fallback (CLI verifier path; the management command does
# not pre-fetch cache_batch_data). Reuse the batch shape so payload +
# etag come from one MGET and stay consistent under concurrent writes.
batch = flags_hypercache.batch_get_from_cache([team])
cached_data, source, cached_etag = batch.get(team.id, (None, "miss", None))
# Get flags from database - use db_batch_data if available to avoid N+1 queries
if db_batch_data and team.id in db_batch_data:
db_data = db_batch_data[team.id]
else:
db_data = _get_feature_flags_for_service(team)
db_flags = db_data.get("flags", []) if isinstance(db_data, dict) else []
# Cache miss. This verifier reads Redis only (via batch_get_from_cache), so in
# practice source is "redis" or "miss"; "db"/"dependency_unavailable" are matched
# defensively to stay in lock-step with verify_team_flag_definitions, which can see
# them through its single-team cold-load fallback. All mean "nothing cached", not drift.
if source in ("db", "miss", "dependency_unavailable"):
return {
"status": "miss",
"issue": "CACHE_MISS",
"details": f"No cache entry found (team has {len(db_flags)} flags in DB)",
"db_data": db_data,
}
# Check if evaluation_metadata is present in cached data.
# Entries written before evaluation_metadata was added won't have it,
# and the Rust service needs it for flag dependency evaluation.
if not cached_data or "evaluation_metadata" not in cached_data:
return {
"status": "mismatch",
"issue": "MISSING_EVALUATION_METADATA",
"details": "Cache entry missing evaluation_metadata",
"db_data": db_data,
}
# Without an etag, the Rust feature-flags in-memory cache bypasses every
# request for this team via the `etag_missing` branch. Surfacing this here
# turns a silent perf regression into a counted verifier mismatch. The
# etag rides on the same MGET as the payload, so this stays O(1) per batch.
# Checked before the per-flag diff: a team with both missing etag AND
# drifted flags reports as MISSING_ETAG; the repair writes db_data back
# and fixes both, so ordering it first skips the diff loop on rollout
# when most teams hit this branch with valid data.
if cached_etag is None:
return {
"status": "mismatch",
"issue": "MISSING_ETAG",
"details": "Cache entry has payload but no etag — Rust in-memory cache will bypass for this team",
"db_data": db_data,
}
# Extract cached flags
cached_flags = cached_data.get("flags", [])
# Compare flags by ID
db_flags_by_id = {flag["id"]: flag for flag in db_flags}
cached_flags_by_id = {flag["id"]: flag for flag in cached_flags}
diffs = []
# Find missing flags (in DB but not in cache)
for flag_id in db_flags_by_id:
if flag_id not in cached_flags_by_id:
diff: dict = {
"type": "MISSING_IN_CACHE",
"flag_id": flag_id,
"flag_key": db_flags_by_id[flag_id].get("key"),
}
diffs.append(diff)
# Find stale flags (in cache but not in DB)
for flag_id in cached_flags_by_id:
if flag_id not in db_flags_by_id:
# An unevaluable cached row is invisible to the matcher, so its presence
# is not drift worth repairing, and reporting it would repair-rewrite
# every old-shape entry at once. A stale ACTIVE flag still reports: its
# cached copy has active=True.
if _is_unevaluable(cached_flags_by_id[flag_id]):
continue
diff = {
"type": "STALE_IN_CACHE",
"flag_id": flag_id,
"flag_key": cached_flags_by_id[flag_id].get("key"),
}
diffs.append(diff)
# Compare field values for flags that exist in both
for flag_id in db_flags_by_id:
if flag_id in cached_flags_by_id:
db_flag = db_flags_by_id[flag_id]
cached_flag = cached_flags_by_id[flag_id]
field_diffs = _compare_flag_fields(db_flag, cached_flag, tolerate_blanked_filters=True)
if field_diffs:
diff = {
"type": "FIELD_MISMATCH",
"flag_id": flag_id,
"flag_key": db_flag.get("key"),
"diff_fields": [f["field"] for f in field_diffs],
}
if verbose:
diff["field_diffs"] = field_diffs
diffs.append(diff)
if not diffs:
return {"status": "match", "issue": "", "details": ""}
# Summarize diffs
missing_count = sum(1 for d in diffs if d.get("type") == "MISSING_IN_CACHE")
stale_count = sum(1 for d in diffs if d.get("type") == "STALE_IN_CACHE")
mismatch_count = sum(1 for d in diffs if d.get("type") == "FIELD_MISMATCH")
summary_parts = []
if missing_count > 0:
summary_parts.append(f"{missing_count} missing")
if stale_count > 0:
summary_parts.append(f"{stale_count} stale")
if mismatch_count > 0:
summary_parts.append(f"{mismatch_count} mismatched")
# Build descriptive diff_flags for logging
diff_flags = []
for d in sorted(diffs, key=lambda x: x.get("flag_key") or str(x["flag_id"])):
flag_key = d.get("flag_key") or str(d["flag_id"])
diff_type = d.get("type")
if diff_type == "MISSING_IN_CACHE":
diff_flags.append(f"{flag_key} {{only in db}}")
elif diff_type == "STALE_IN_CACHE":
diff_flags.append(f"{flag_key} {{only in cache}}")
elif diff_type == "FIELD_MISMATCH":
fields = d.get("diff_fields", [])
diff_flags.append(f"{flag_key} {{fields: {', '.join(fields)}}}")
result: dict = {
"status": "mismatch",
"issue": "DATA_MISMATCH",
"details": f"{', '.join(summary_parts)} flags" if summary_parts else "unknown differences",
"diff_flags": diff_flags,
"db_data": db_data,
}
if verbose:
result["diffs"] = diffs
return result
# Keys whose ``null`` is semantically distinct from "absent" and must be preserved
# during loose comparison — but only at the group level (``filters.groups[*]`` /
# ``filters.super_groups[*]``), where the Rust matcher uses
# ``Option<Option<i32>>`` with ``skip_serializing_if`` on
# ``FlagPropertyGroup.aggregation_group_type_index`` to distinguish "absent
# (fall back to flag-level group type)" from "explicit null (force person-level
# aggregation)". At the filters level the Rust ``FlagFilters`` field is plain
# ``Option<i32>`` with no ``skip_serializing_if``, so the warmer always emits
# ``null`` for both PG-null and PG-absent; preserving null there would flag
# every PG-absent team as drifted. The list keys below mark the path into
# group-level dicts so preservation only kicks in for those nested entries.
_PRESERVE_NULL_KEYS = frozenset({"aggregation_group_type_index"})
_GROUP_LEVEL_LIST_KEYS = frozenset({"groups", "super_groups"})
def _strip_null_values(value: Any, in_group_level_list: bool = False) -> Any:
"""Recursively drop ``None`` values from dicts; recurse into lists without dropping ``None`` elements.
Used to normalize structural divergence between the Django serializer (which
passes JSONB through verbatim and preserves explicit ``null`` entries) and
the Rust warmer (whose typed ``Option<T>`` deserialization collapses
"absent key" and "explicit null" into the same ``None`` and emits one
shape). The matcher already treats those two states as equivalent for most
fields, so the verifier should mirror that tolerance instead of reporting
spurious ``FIELD_MISMATCH``.
Rules:
- Dicts: drop entries whose value is ``None``, except for keys in
``_PRESERVE_NULL_KEYS`` when this dict is a group-level item (sits inside
``filters.groups[*]`` or ``filters.super_groups[*]``), where ``null`` is
semantically distinct from absent; recurse into remaining values, marking
group-level list values so their dict elements know they're at group level.
- Lists: recurse into each element, but preserve ``None`` elements (dropping
them would shift indices and change semantics). The Rust typed
serialization will not emit ``null`` list elements in current data, so
this rule preserves correctness without changing observed behavior.
- Scalars: returned unchanged.
See plans/verify-flags-cache-loose-comparison.md for the full rationale.
"""
if isinstance(value, dict):
return {
k: _strip_null_values(v, in_group_level_list=(k in _GROUP_LEVEL_LIST_KEYS))
for k, v in value.items()
if v is not None or (in_group_level_list and k in _PRESERVE_NULL_KEYS)
}
if isinstance(value, list):
return [_strip_null_values(item, in_group_level_list=in_group_level_list) for item in value]
return value
def _compare_flag_fields(db_flag: dict, cached_flag: dict, *, tolerate_blanked_filters: bool = False) -> list[dict]:
"""Compare field values between DB and cached versions of a flag.
The DB serialization is treated as the source of truth: only keys present in
``db_flag`` are compared. Extra keys in ``cached_flag`` (e.g. fields that
were removed from the serializer but still linger in pre-existing cache
entries) are ignored so that benign serializer field removals do not flag
every team's cache as mismatched.
For container values (dicts and lists, which is where every observed
absent/null divergence lives — under ``filters``), both sides are passed
through ``_strip_null_values`` before the equality check so that explicit
``null`` and absent-key normalize to the same shape. Top-level scalar keys
are compared directly; ``MinimalFeatureFlagSerializer`` emits all top-level
keys explicitly today, so there is no top-level absent/null divergence to
tolerate. See plans/verify-flags-cache-loose-comparison.md.
``tolerate_blanked_filters`` exempts ``filters`` when both sides agree the flag
is unevaluable. Only the ``flags.json`` writers blank those filters, and entries
predating that still hold the full blob while the two writers deploy
independently, so the blob and ``{"groups": []}`` both occur and the matcher
ignores either one. It is opt-in because ``local_evaluation`` shares this
function for ``flags_with_cohorts.json``, which has a single writer and no
blanking: there any ``filters`` difference is real drift worth repairing. A
disagreement about ``active`` itself is reported in both modes.
"""
field_diffs = []
# Mirrors the writers' own predicate so a flag they blank is never reported as
# unfixable drift.
both_unevaluable = tolerate_blanked_filters and _is_unevaluable(db_flag) and _is_unevaluable(cached_flag)
for key in db_flag.keys():
if key == "filters" and both_unevaluable:
continue
db_val = db_flag[key]
cached_val = cached_flag.get(key)
if isinstance(db_val, dict | list) or isinstance(cached_val, dict | list):
if _strip_null_values(db_val) != _strip_null_values(cached_val):
field_diffs.append({"field": key, "db_value": db_val, "cached_value": cached_val})
elif db_val != cached_val:
field_diffs.append({"field": key, "db_value": db_val, "cached_value": cached_val})
return field_diffs
def get_teams_with_flags_queryset() -> "QuerySet[Team]":
"""
Return a queryset of teams that have ever had a feature flag.
Queries via ``objects_including_soft_deleted`` so that teams whose flags
were all soft-deleted still get their cache verified (the cache should
contain ``{"flags": []}``, not be absent).
Used as the single source of truth for scoping both Celery verification
tasks and management commands to the ~10% of teams that have flags.
"""
# Use Q() to pass team_id as a positional arg, bypassing RootTeamQuerySet.filter()
# which intercepts team_id kwargs and adds expensive parent-team JOIN/subquery logic
# that makes the correlated EXISTS subquery unusable at scale.
has_flags = FeatureFlag.objects_including_soft_deleted.filter(Q(team_id=OuterRef("pk")))
return Team.objects.filter(Exists(has_flags))
def get_team_ids_with_recently_updated_flags(team_ids: list[int]) -> set[int]:
"""
Batch check which teams have active flags updated within the grace period.
When a flag is updated, an async task updates the cache. If verification
runs before the async task completes, it sees a stale cache and tries to
"fix" it, causing unnecessary work. This grace period lets recent async
updates complete before treating cache misses as genuine errors.
Only considers active, non-deleted flags. When a flag is deleted or
deactivated, the cache update removes it, so we shouldn't skip verification
just because a deleted/inactive flag was recently updated.
Args:
team_ids: List of team IDs to check
Returns:
Set of team IDs that have recently updated active flags (should skip fix)
"""
grace_period_minutes = settings.FLAGS_CACHE_VERIFICATION_GRACE_PERIOD_MINUTES
if grace_period_minutes <= 0 or not team_ids:
return set()
cutoff = timezone.now() - timedelta(minutes=grace_period_minutes)
return set(
FeatureFlag.objects.filter(team_id__in=team_ids, updated_at__gte=cutoff, active=True)
.values_list("team_id", flat=True)
.distinct()
)
# Initialize hypercache management config after update_flags_cache is defined
FLAGS_HYPERCACHE_MANAGEMENT_CONFIG = HyperCacheManagementConfig(
hypercache=flags_hypercache,
update_fn=update_flags_cache,
cache_name="flags",
get_teams_queryset_fn=get_teams_with_flags_queryset,
get_team_ids_to_skip_fix_fn=get_team_ids_with_recently_updated_flags,
# Late-bound via lambda: the attribution fn lives in the transitional
# Kafka-routing block further down this module and is deleted with it at
# cutover, so it isn't defined yet when this config is constructed.
get_primary_writer_fn=lambda team_id: get_team_primary_flags_writer(team_id),
# Late-bound for the same reason as get_primary_writer_fn above.
route_refresh_fn=lambda team_id: route_refresh_to_kafka(team_id),
# The refresh loads flags by team id/project_id; it reads no other Team columns.
# Narrowing the SELECT keeps it resilient to newly added Team columns the read
# replica may not have applied yet (organization_id keeps the select_related valid).
refresh_only_fields=["id", "project_id", "organization_id"],
)
def clear_flags_cache(team: Team | int, kinds: list[str] | None = None) -> None:
"""
Clear the flags cache for a team.
Only operates when FLAGS_REDIS_URL is configured to avoid writing to shared cache.
Args:
team: Team object or team ID
kinds: Optional list of cache kinds to clear ("redis", "s3")
"""
if not settings.FLAGS_REDIS_URL:
return
flags_hypercache.clear_cache(team, kinds=kinds)
def get_teams_with_expiring_flags_caches(ttl_threshold_hours: int = 24, limit: int = 5000) -> list[Team]:
"""
Get teams whose flags caches are expiring soon using sorted set for efficient lookup.
Uses ZRANGEBYSCORE on the expiry tracking sorted set instead of scanning all Redis keys.
This is O(log N + M) where M is the number of expiring teams, vs O(N) for SCAN.
Args:
ttl_threshold_hours: Refresh caches expiring within this many hours
limit: Maximum number of teams to return (default 5000)
Returns:
List of Team objects whose caches need refresh (up to limit)
"""
return get_teams_with_expiring_caches(FLAGS_HYPERCACHE_MANAGEMENT_CONFIG, ttl_threshold_hours, limit)
def refresh_expiring_flags_caches(ttl_threshold_hours: int = 24, limit: int = 5000) -> CacheRefreshCounts:
"""
Refresh flags caches that are expiring soon to prevent cache misses.
This is the main hourly job that keeps caches fresh. It:
1. Finds cache entries with TTL < threshold (up to limit)
2. Refreshes them with new data and full TTL, or, for a team the refresh routing
flag has claimed, raises a Kafka invalidation and leaves the build to the Rust
builder instead
Processes teams in batches (default 5000). If more teams are expiring than the limit,
subsequent runs will process the next batch.
Note: Metrics are pushed to Pushgateway by refresh_expiring_caches() via push_hypercache_teams_processed_metrics()
Args:
ttl_threshold_hours: Refresh caches expiring within this many hours
limit: Maximum number of teams to refresh per run (default 5000)
5000 chosen as starting point to balance:
- Memory efficiency: Doesn't load too many teams into memory at once
- Throughput: With ~200K teams total, hourly runs can process 120K/day (5000 * 24)
- Responsiveness: Completes quickly enough to not block other operations
Returns:
CacheRefreshCounts with successful, failed and enqueued refresh counts
"""
return refresh_expiring_caches(
FLAGS_HYPERCACHE_MANAGEMENT_CONFIG,
ttl_threshold_hours,
limit,
pacing=RefreshPacing(
chunk_size=settings.FLAGS_CACHE_REFRESH_KAFKA_CHUNK_SIZE,
delay_seconds=settings.FLAGS_CACHE_REFRESH_KAFKA_CHUNK_DELAY_SECONDS,
window_seconds=settings.FLAGS_CACHE_REFRESH_KAFKA_WINDOW_SECONDS,
),
)
def cleanup_stale_expiry_tracking() -> int:
"""
Clean up orphaned entries in the expiry tracking sorted set.
Removes entries for teams that no longer exist in the database.
Should be run periodically (e.g., daily) to prevent sorted set bloat.
Returns:
Number of stale entries removed
"""
removed = cleanup_generic(FLAGS_HYPERCACHE_MANAGEMENT_CONFIG)
if removed > 0:
TOMBSTONE_COUNTER.labels(
namespace="flags",
operation="stale_expiry_tracking",
component="flags_cache",
).inc(removed)
return removed
def get_cache_stats() -> dict[str, Any]:
"""
Get statistics about the flags cache.
Returns:
Dictionary with cache statistics including size information
"""
return get_cache_stats_generic(FLAGS_HYPERCACHE_MANAGEMENT_CONFIG)
# Signal handlers for automatic cache invalidation
# KAFKA-CUTOVER TRANSITIONAL CODE — remove at cutover.
# Stages: producer (this block) at 0% → Rust consumer ships → ramp gate to 100%
# → Kafka becomes primary, this block is deleted and the signal handlers call
# the Kafka path directly. The Celery task `update_team_service_flags_cache`
# outlives cutover — `cohort_changed_flags_cache` still calls it directly until
# cohort invalidation gets its own topic. Throwaway code by design; don't polish.
#
# Transitional surface: KAFKA_ROUTING_FLAG, _evaluate_kafka_routing_flag,
# _route_to_kafka, get_team_primary_flags_writer (and its config binding on
# FLAGS_HYPERCACHE_MANAGEMENT_CONFIG), route_refresh_to_kafka (and its config
# binding), SHADOW_COMPARE_FLAG, _shadow_compare_enabled, publish_shadow_invalidation,
# _produce_invalidation, _enqueue_invalidation, and the Kafka branch inside it.
# The signal handlers themselves stay; their tails simplify at cutover. The one
# call site outside this block is the tail of update_team_service_flags_cache in
# tasks.py, which goes with it.
#
# At cutover the sweep routes unconditionally: the FLAGS_CACHE_REFRESH_KAFKA_ENABLED
# guard goes, and the binding is replaced by a hook that always produces, not deleted.
# The generic surface it uses — route_refresh_fn, the enqueued count and RefreshPacing —
# keeps its caller and stays.
# Per-team gate that routes invalidation to Kafka instead of Celery — see
# _enqueue_invalidation for why the two paths are mutually exclusive. The key
# string is kept as "dual-write" (not renamed to match KAFKA_ROUTING_FLAG) since
# it's the live PostHog flag key — renaming it would repoint the rollout.
KAFKA_ROUTING_FLAG = "flags-cache-kafka-dual-write"
def _evaluate_kafka_routing_flag(team_id: int) -> bool | None:
# The SDK annotates feature_enabled as returning bool, but it returns
# None when local evaluation can't resolve the flag. Widen the type so
# callers can handle the None branch under type checking.
result: bool | None = posthoganalytics.feature_enabled(
KAFKA_ROUTING_FLAG,
f"team-{team_id}",
groups={"project": str(team_id)},
group_properties={"project": {"id": str(team_id)}},
only_evaluate_locally=True,
send_feature_flag_events=False,
)
return result
def get_team_primary_flags_writer(team_id: int) -> str:
"""Which writer owns this team's flags.json entry: "rust" when the routing
flag sends its invalidations to the Kafka builder, otherwise "python".
Attributes verifier fixes to the writer whose output needed fixing. During
the Kafka-builder ramp, a fix on a rust-routed team is the signal that the
Rust builder diverged from the Python one, which the unattributed fix
counter cannot separate from baseline repair noise. Evaluation failures
resolve to "unknown" rather than "python" so an attribution outage cannot
masquerade as a clean Rust ramp.
"""
try:
result = _evaluate_kafka_routing_flag(team_id)
except SoftTimeLimitExceeded:
# A Celery soft-time-limit is a task-control signal, not an attribution
# failure. Let it propagate so the verify sweep winds down, matching the
# other SoftTimeLimitExceeded guards in the verifier.
raise
except Exception:
# Mirror _route_to_kafka's evaluation-failure logging so a broken
# attribution client is diagnosable in Sentry, because otherwise it only
# shows up as a rise in writer="unknown" that cannot be told apart from a
# cold local flag cache (which returns None below and is expected at boot).
logger.warning(
"flags_cache_writer_attribution_flag_evaluation_failed",
team_id=team_id,
flag=KAFKA_ROUTING_FLAG,
exc_info=True,
)
return "unknown"
if result is None:
return "unknown"
return "rust" if result else "python"
def _route_to_kafka(team_id: int) -> bool:
"""Return True if this team's invalidation should route to Kafka instead of Celery.
A `None` return from `feature_enabled` means the local-eval cache hasn't
loaded the flag definition yet. Treated as disabled, but ticks
TOMBSTONE_COUNTER so a fleet-wide silent disable (polling thread wedged)
is visible on existing Grafana dashboards: a short burst at boot is
expected; a sustained non-zero rate means polling is broken.
"""
try:
result = _evaluate_kafka_routing_flag(team_id)
except Exception:
# If the flag client misbehaves, default to Celery-only — never block the signal handler.
# Log so a silent disable across the fleet during rollout is visible in Sentry.
logger.warning(
# Event name kept as "dual_write" (not renamed to match KAFKA_ROUTING_FLAG) so it
# keeps matching existing Sentry searches set up during the dual-write phase.
"flags_cache_kafka_dual_write_flag_evaluation_failed",
team_id=team_id,
flag=KAFKA_ROUTING_FLAG,
exc_info=True,
)
return False
if result is None:
TOMBSTONE_COUNTER.labels(
namespace="flags",
# Label kept as "dual_write" for continuity with the existing Grafana dashboards
# referenced above.
operation="dual_write_gate_cache_cold",
component="flags_cache",
).inc()
return False
return bool(result)
def route_refresh_to_kafka(team_id: int) -> bool:
"""Routing hook for the expiry refresh sweep, bound on
FLAGS_HYPERCACHE_MANAGEMENT_CONFIG.
Returns True when the team's refresh was raised as a Kafka invalidation, which
makes the sweep skip its own build and count the team as enqueued.
A produce failure propagates instead of returning True, so the sweep counts the
team as failed rather than enqueued. That covers what `produce` reports
synchronously: a full local queue, a serialization error, and a producer that
cannot be constructed. It does not cover broker-side loss, because `flush_timeout=0`
returns before any ack, so an unreachable cluster still queues the message locally
and the team reads as enqueued. The expiry backlog gauge and the verifier are what
catch that class, never this count.
A propagated error is still not a Celery fallback, for the reason
`_enqueue_invalidation` gives: the two paths are mutually exclusive so a broken
Kafka path shows up as a stale cache instead of being masked by Python quietly
building the team anyway. The sweep skips its own build for a team whose hook
failed. A team whose message is lost stays inside the expiry window, so the next
hourly run raises it again, and the verifier repairs it in the meantime.
FLAGS_CACHE_REFRESH_KAFKA_ENABLED is read per deployment, so each region turns
routing on only after its flags-cache-builder image accepts the `source` field.
An older builder rejects the message through deny_unknown_fields, counts it as a
parse error and drops it, and the dead letter queue holds build failures rather
than parse failures, so nothing can be replayed. With the setting off the sweep
keeps building in Python, which is the safe direction: a build that happens is
never worse than a message nobody consumes.
"""
if not settings.FLAGS_CACHE_REFRESH_KAFKA_ENABLED:
return False
_produce_invalidation(team_id, source="refresh", raise_on_error=True)
return True
# Per-team gate for shadow parity publishing. A different flag from
# KAFKA_ROUTING_FLAG on purpose: this one never decides which writer serves a
# team, so the two ramp independently. See flags_cache_messages for what the
# consumer does with a shadow message.
SHADOW_COMPARE_FLAG = "flags-cache-shadow-compare"
def _shadow_compare_enabled(team_id: int) -> bool:
"""Return True if shadow parity comparison is enabled for this team.
Every failure mode returns False, because shadow publishing is telemetry: an
outage here must cost parity evidence and never a cache build.
Unlike `_route_to_kafka`, an unresolvable flag ticks no TOMBSTONE_COUNTER.
Local evaluation cannot resolve SHADOW_COMPARE_FLAG for as long as the flag
does not exist in PostHog, so a tick would fire on every Celery-owned rebuild,
holding a constant high rate on a panel that means "rare anomaly". A client
that is actually broken raises instead, and `publish_shadow_invalidation`
logs that.
"""
try:
return feature_enabled_or_false(
SHADOW_COMPARE_FLAG,
f"team-{team_id}",
groups={"project": str(team_id)},
group_properties={"project": {"id": str(team_id)}},
only_evaluate_locally=True,
send_feature_flag_events=False,
)
except Exception:
# Log so a fleet-wide silent disable is visible in Sentry rather than
# showing up only as parity evidence that never arrives.
logger.warning(
"flags_cache_shadow_compare_flag_evaluation_failed",
team_id=team_id,
flag=SHADOW_COMPARE_FLAG,
exc_info=True,
)
return False
def publish_shadow_invalidation(team_id: int) -> None:
"""Publish a parity-telemetry message for a team the Celery builder just rebuilt.
Called from the tail of `update_team_service_flags_cache` rather than from
`_enqueue_invalidation`, so the Rust builder diffs its build against a cache
entry Python has already written. Publishing at invalidation time races that
rebuild: the builder reads fresh Postgres against a pre-rebuild Redis entry
and reports the gap as drift. The consumer suppresses a one-off mismatch by
content hash, but two invalidations that both land inside one stale window
hash the same, which promotes pure Celery lag to a confirmed parity defect.
The Kafka-routing decision still applies, because this task also serves teams
Rust owns: `cohort_changed_flags_cache` and the cohort backfill finalizer
dispatch it for every team, and a shadow build for a Rust-owned team would
diff the Rust output against itself.
It reads `_evaluate_kafka_routing_flag` rather than `_route_to_kafka` so those
callers do not start ticking the routing gate's cold-cache tombstone counter.
Neither of them consulted that gate before, so a backfill on a freshly booted
worker would otherwise spike a panel that means the flag polling thread is
wedged.
Both steps swallow their own failures, so a shadow publish cannot fail the
build that precedes it.
"""
try:
routed_to_kafka = _evaluate_kafka_routing_flag(team_id)
except Exception:
# A raising client means something is broken, unlike an unresolved flag,
# which is routine while SHADOW_COMPARE_FLAG does not exist. This is the
# only signal that shadow volume went to zero on the worker, because the
# web process runs a separate client and reports under its own event.
logger.warning(
"flags_cache_shadow_routing_evaluation_failed",
team_id=team_id,
flag=KAFKA_ROUTING_FLAG,
exc_info=True,
)
return
# Only an explicit False means Celery owns this team. `None` means local
# evaluation cannot resolve ownership, and shadow-building a team Rust owns
# would diff the Rust output against itself.
if routed_to_kafka is not False:
return
if _shadow_compare_enabled(team_id):
_produce_invalidation(team_id, shadow=True)
def _produce_invalidation(
team_id: int,
shadow: bool = False,
source: Literal["edit", "refresh"] = "edit",
raise_on_error: bool = False,
) -> None:
"""Produce a single invalidation message; log Kafka errors, and propagate them
only for a caller that reports them.
A produce failure must not raise out of a signal handler and is deliberately
not retried via Celery (see `_enqueue_invalidation`). The `except` below only
sees synchronous errors (`BufferError` queue-full, serialization/config).
`flush_timeout=0` returns without waiting for acks rather than stalling
every flag-edit on-commit hook on an unhealthy cluster, so broker-side
failures surface later in `_KafkaProducer._on_delivery` as a counter tick
plus a throttled `kafka_producer_delivery_failed` warning keyed by team id.
`data` must be a dict, not pre-encoded bytes: `produce` runs it through
`json.dumps`, so bytes would `TypeError` and silently fail the swallow path.
`mode="json"` converts `datetime` to ISO string.
`source` names the producer that raised the message. The builder treats every
source identically and uses the value to attribute builds and latency, so a wrong
value costs a reading and never a build. See flags_cache_messages for why a region
whose builder predates the field must not be sent anything but the default.
`raise_on_error` belongs to the caller, not to the wire: it says whether a caller
is there to report the failure. The refresh sweep sets it so a team the `except`
below catches counts as failed instead of as a hand-off that never happened. It
reaches only that synchronous class, never a delivery failure, which arrives after
this function has returned. Signal handlers leave it off, because a flag edit has
nothing to report a produce failure to.
"""
try:
msg = FlagsCacheInvalidation(team_id=team_id, emitted_at=datetime.now(UTC), shadow=shadow, source=source)
with producer_scope(topic=KAFKA_FLAGS_CACHE_INVALIDATION, flush_timeout=0) as producer:
producer.produce(
topic=KAFKA_FLAGS_CACHE_INVALIDATION,
data=msg.model_dump(mode="json"),
key=str(team_id),
# The key is only ever a team id, so it is safe to log.
log_key_on_delivery_failure=True,
)
except Exception as e:
logger.warning(
"flags_cache_invalidation_produce_failed",
team_id=team_id,
shadow=shadow,
source=source,
error=str(e),
exc_info=True,
)
if raise_on_error:
raise
def _enqueue_invalidation(team_id: int) -> None:
"""Run from `transaction.on_commit`: route to Kafka if enabled, otherwise Celery.
Signal handlers defer this until commit so the Celery worker can't read
pre-commit state; callers with no open transaction (staff tooling, via
`enqueue_evaluation_cache_invalidation`) call it directly. Cohort
invalidation stays on its own topic.
The two paths are mutually exclusive so the rollout proves the Kafka path
end to end: a broken Kafka path shows up as a stale cache instead of being
masked by Celery quietly picking up the slack. Do not add a Celery fallback
on produce failure. It would hide that signal, and it would miss the
realistic failure mode anyway, because broker-side delivery failures are
reported only after `_produce_invalidation` has returned (see its
docstring). A dropped invalidation is repaired by the
`verify_and_fix_flags_cache_task` sweep (which includes teams whose flags
are all soft-deleted), so worst-case staleness is one sweep interval plus
the verification grace period, well under an hour rather than the cache
TTL. Celery's `.delay()` may raise when the flag is off, since it is the
sole path then and operators want broker failures loud.
Shadow parity messages are not produced here. They are produced after the
Celery build writes the cache, in `publish_shadow_invalidation`, which
explains why.
Guarded on FLAGS_REDIS_URL here rather than at each call site so every
caller gets the same no-op-when-unconfigured behavior.
"""
if not settings.FLAGS_REDIS_URL:
return
from products.feature_flags.backend.tasks import update_team_service_flags_cache
if _route_to_kafka(team_id):
_produce_invalidation(team_id)
else:
update_team_service_flags_cache.delay(team_id)
def enqueue_evaluation_cache_invalidation(team_id: int) -> None:
"""Public entry point for `_enqueue_invalidation`, for callers outside a model signal handler
(e.g. staff tooling) that want a rebuild to raise the exact same invalidation signal (Kafka
or Celery routing) that an organic flag create/update/delete raises."""
_enqueue_invalidation(team_id)
# The two caches a cohort save rebuilds: the local-evaluation definitions cache and the
# flags-service cache.
FlagsCacheName = Literal["definitions", "service"]
_deferred_rebuild_teams: ContextVar[dict[int, set[FlagsCacheName]] | None] = ContextVar(
"deferred_flags_cache_rebuild_teams", default=None
)
def defer_flags_cache_rebuild(team_id: int, cache: FlagsCacheName) -> bool:
"""Record the team's rebuild of `cache` if a coalescing block is active.
Returns True when the rebuild was recorded, which means the caller must not enqueue its
own.
"""
teams = _deferred_rebuild_teams.get()
if teams is None:
return False
teams.setdefault(team_id, set()).add(cache)
return True
def _dispatch_flags_cache_rebuilds(team_id: int, caches: set[FlagsCacheName], failed_teams: set[int]) -> None:
"""Enqueue the recorded rebuilds for one team, recording the team when a publish fails."""
from products.feature_flags.backend.tasks import update_team_flags_cache, update_team_service_flags_cache
tasks = {"definitions": update_team_flags_cache, "service": update_team_service_flags_cache}
failed = False
for cache in sorted(caches):
# One try per task, so a broker failure on one does not drop the other.
try:
tasks[cache].delay(team_id)
except Exception:
# The saves already committed, so raising would abort the rest of a fleet-wide run.
# The caller reports the team instead.
failed = True
logger.error("coalesced_flags_cache_dispatch_failed", team_id=team_id, cache=cache, exc_info=True)
if failed:
failed_teams.add(team_id)
else:
logger.info("coalesced_flags_cache_dispatch", team_id=team_id)
@contextmanager
def coalesced_cohort_flags_cache_rebuilds() -> Iterator[set[int]]:
"""Coalesce the cohort-save flags-cache rebuilds inside the block to one dispatch per team.
A bulk cohort resave otherwise enqueues two whole-team rebuilds per cohort, where two per
team are sufficient. Only the two cohort receivers coalesce; flag and experiment saves
inside the block still enqueue their own rebuild. Each receiver records the cache it was
about to rebuild, so the dispatch enqueues exactly what the receivers wanted.
Yields the set of team ids whose dispatch failed, so the caller can report them. In
autocommit it is filled by the time the block exits; under an open transaction the dispatch
waits for the commit and the set fills later.
"""
failed_teams: set[int] = set()
token = _deferred_rebuild_teams.set({})
try:
yield failed_teams
finally:
teams = _deferred_rebuild_teams.get() or {}
# Reset before dispatching: eager Celery runs the rebuild inline, and a cohort save
# inside it would otherwise record into a dict nobody reads again.
_deferred_rebuild_teams.reset(token)
for team_id, caches in sorted(teams.items()):
transaction.on_commit(partial(_dispatch_flags_cache_rebuilds, team_id, caches, failed_teams))
@receiver(post_save, sender=FeatureFlag)
@receiver(post_delete, sender=FeatureFlag)
def feature_flag_changed_flags_cache(sender, instance: "FeatureFlag", **kwargs):
"""
Invalidate flags cache when a feature flag is created, updated, or deleted.
This ensures the feature-flags service always has fresh flag data after any flag changes.
Only operates when FLAGS_REDIS_URL is configured.
"""
if not settings.FLAGS_REDIS_URL:
return
team_id = instance.team_id
transaction.on_commit(lambda: _enqueue_invalidation(team_id))
@receiver(post_save, sender=Experiment)
@receiver(post_delete, sender=Experiment)
def experiment_changed_flags_cache(sender, instance: "Experiment", **kwargs):
"""
Invalidate flags cache when an experiment is created, soft-deleted, or removed.
A flag's cached `has_experiment` depends on whether it has any non-deleted linked
experiment, so experiment changes must refresh the linked flag's team cache.
Keyed on the experiment's own team_id, which also covers experiment reassignment
between flags within the team. Only operates when FLAGS_REDIS_URL is configured.
Fires on every save by design, mirroring feature_flag_changed_flags_cache: Experiment
rows are only written on user-driven lifecycle/edit operations (no high-churn periodic
path touches them, unlike cohort recalculation), so an update_fields gate isn't warranted.
"""
if not settings.FLAGS_REDIS_URL:
return
team_id = instance.team_id
transaction.on_commit(lambda: _enqueue_invalidation(team_id))
@receiver(post_save, sender=Team)
def team_created_flags_cache(sender, instance: "Team", created: bool, **kwargs):
"""
Warm flags cache when a team is created.
This ensures the cache is immediately available for new teams.
Only operates when FLAGS_REDIS_URL is configured.
"""
if not created or not settings.FLAGS_REDIS_URL:
return
team_id = instance.id
transaction.on_commit(lambda: _enqueue_invalidation(team_id))
@receiver(post_delete, sender=Team)
def team_deleted_flags_cache(sender, instance: "Team", **kwargs):
"""
Clear flags cache when a team is deleted.
This ensures we don't have stale cache entries for deleted teams.
Only operates when FLAGS_REDIS_URL is configured.
"""
if not settings.FLAGS_REDIS_URL:
return
# For unit tests, only clear Redis to avoid S3 timestamp issues with frozen time
kinds = ["redis"] if settings.TEST else None
clear_flags_cache(instance, kinds=kinds)
@receiver(post_save, sender=FeatureFlagEvaluationContext)
@receiver(post_delete, sender=FeatureFlagEvaluationContext)
def evaluation_context_changed_flags_cache(sender, instance: "FeatureFlagEvaluationContext", **kwargs):
"""
Invalidate flags cache when evaluation contexts are added or removed from a flag.
Evaluation context names are cached as part of the flag data, so changes to the
FeatureFlagEvaluationContext join table require a cache refresh.
Only operates when FLAGS_REDIS_URL is configured.
"""
if not settings.FLAGS_REDIS_URL:
return
team_id = instance.feature_flag.team_id
transaction.on_commit(lambda: _enqueue_invalidation(team_id))
@receiver(post_save, sender=Cohort)
@receiver(post_delete, sender=Cohort)
def cohort_changed_flags_cache(sender, instance: "Cohort", **kwargs):
"""
Invalidate flags cache when a cohort definition changes.
Skips recalculation-only saves (count, version, is_calculating, etc.) to avoid
rebuilding the flags cache on every static cohort recalculation.
Only operates when FLAGS_REDIS_URL is configured.
"""
if not settings.FLAGS_REDIS_URL:
return
update_fields = kwargs.get("update_fields")
if update_fields is not None and frozenset(update_fields) <= _COHORT_RECALCULATION_FIELDS:
return
if defer_flags_cache_rebuild(instance.team_id, "service"):
return
from products.feature_flags.backend.tasks import update_team_service_flags_cache
# Intentionally bypasses _enqueue_invalidation: cohort changes do not
# share the flag-invalidation Kafka topic.
transaction.on_commit(lambda: update_team_service_flags_cache.delay(instance.team_id))