Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): clear local evaluation cache when a team is deleted (#128011)

Ben White0c4ff77a5bf2
diff --git a/docs/internal/feature-flags/hypercache-system.md b/docs/internal/feature-flags/hypercache-system.md
index e21e8bb19593..be7da8e895a7 100644
--- a/docs/internal/feature-flags/hypercache-system.md
+++ b/docs/internal/feature-flags/hypercache-system.md
@@ -234,6 +234,12 @@ Models that invalidate the flags cache:
- `FeatureFlagEvaluationTag` - Evaluation tags changed
- `Tag` - Tag renamed
+Deleting a `Team` clears its local evaluation entry from Redis and S3 after the deletion commits.
+The rebuild task cannot run for a deleted team, and `/flags/definitions` falls back to the S3 copy, which has no TTL.
+
+The staff cache tools `clear` action also accepts the ids of deleted teams.
+Use it to purge an orphaned entry for a team that no longer exists.
+
## Remote config caching
Remote config uses a different caching strategy than local evaluation. It prioritizes zero-overhead cache hits by using Redis-only caching for serving, with HyperCache used only for background sync operations.
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index d2947215f27d..1cf302afc5bf 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -118,6 +118,7 @@ post_delete:products.feature_flags.backend.local_evaluation.cohort_changed
post_delete:products.feature_flags.backend.local_evaluation.evaluation_context_changed
post_delete:products.feature_flags.backend.local_evaluation.experiment_changed
post_delete:products.feature_flags.backend.local_evaluation.feature_flag_changed
+post_delete:products.feature_flags.backend.local_evaluation.team_deleted
post_delete:products.managed_warehouse.backend.models._observe_duckgres_server_delete
post_delete:products.replay_vision.backend.media_expiry.expire_media_asset
post_delete:products.review_hog.backend.ownership._invalidate_owned_repositories_on_delete
diff --git a/products/feature_flags/backend/api/staff_cache.py b/products/feature_flags/backend/api/staff_cache.py
index d3ef5d230548..8075c495834f 100644
--- a/products/feature_flags/backend/api/staff_cache.py
+++ b/products/feature_flags/backend/api/staff_cache.py
@@ -312,17 +312,20 @@ def _dispatch_mutation(
evaluation_fn: Callable[[int], None],
definitions_fn: Callable[[int], None],
log_event: str,
+ queue_missing_teams: bool = False,
) -> response.Response:
"""Shared shape of `rebuild` and `clear`: split team_ids into found/not-found, dispatch the
- per-cache action for each found team, log, and return the 202 response both actions share."""
+ per-cache action for each found team (or every team with `queue_missing_teams`), log, and
+ return the 202 response both actions share."""
# Dedupe (preserving order) so a caller passing the same id twice doesn't enqueue duplicate
# rebuild/clear work for that team.
deduped_ids = list(dict.fromkeys(team_ids))
found_set = set(Team.objects.filter(id__in=deduped_ids).values_list("id", flat=True))
found_ids = [team_id for team_id in deduped_ids if team_id in found_set]
not_found_ids = [team_id for team_id in deduped_ids if team_id not in found_set]
+ queued_ids = deduped_ids if queue_missing_teams else found_ids
- for team_id in found_ids:
+ for team_id in queued_ids:
if EVALUATION in caches:
evaluation_fn(team_id)
if DEFINITIONS in caches:
@@ -332,13 +335,13 @@ def _dispatch_mutation(
log_event,
staff_user_id=request.user.id,
was_impersonated=is_impersonated(request),
- team_ids=found_ids,
+ team_ids=queued_ids,
not_found_team_ids=not_found_ids,
caches=caches,
)
return response.Response(
- {"queued_team_ids": found_ids, "not_found_team_ids": not_found_ids},
+ {"queued_team_ids": queued_ids, "not_found_team_ids": not_found_ids},
status=status.HTTP_202_ACCEPTED,
)
@@ -506,4 +509,6 @@ class FeatureFlagsStaffCacheViewSet(viewsets.ViewSet):
evaluation_fn=clear_team_evaluation_cache.delay,
definitions_fn=clear_team_definitions_cache.delay,
log_event="flags_staff_cache_clear",
+ # Clearing needs no Team row, so this also purges entries a deleted team left behind.
+ queue_missing_teams=True,
)
diff --git a/products/feature_flags/backend/local_evaluation.py b/products/feature_flags/backend/local_evaluation.py
index da6fb761fd48..fa6b587c7363 100644
--- a/products/feature_flags/backend/local_evaluation.py
+++ b/products/feature_flags/backend/local_evaluation.py
@@ -988,6 +988,16 @@ def feature_flag_changed(sender, instance: "FeatureFlag", **kwargs):
transaction.on_commit(lambda: update_team_flags_cache.delay(instance.team_id))
+@receiver(post_delete, sender=Team)
+def team_deleted(sender, instance: Team, **kwargs) -> None:
+ # The S3 copy has no TTL, and /flags/definitions falls back to it when Redis misses,
+ # so without this a deleted team's definitions stay served indefinitely.
+ from products.feature_flags.backend.tasks import clear_team_definitions_cache
+
+ team_id = instance.id
+ transaction.on_commit(lambda: clear_team_definitions_cache.delay(team_id))
+
+
@receiver(post_save, sender=Experiment)
@receiver(post_delete, sender=Experiment)
def experiment_changed(sender, instance: "Experiment", **kwargs):
diff --git a/products/feature_flags/backend/tasks.py b/products/feature_flags/backend/tasks.py
index ef465c3fe7ea..3a89c1eb76e1 100644
--- a/products/feature_flags/backend/tasks.py
+++ b/products/feature_flags/backend/tasks.py
@@ -35,7 +35,7 @@ from products.feature_flags.backend.flags_cache import (
)
from products.feature_flags.backend.local_evaluation import (
FLAG_DEFINITIONS_HYPERCACHE_MANAGEMENT_CONFIG,
- clear_flag_definition_caches,
+ flag_definitions_hypercache,
update_flag_caches,
)
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -234,11 +234,19 @@ def clear_team_evaluation_cache(team_id: int) -> None:
clear_flags_cache(team_id)
-@shared_task(ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS.value)
+# The task retries because no rebuild can repair the entry once the Team row is gone.
+@shared_task(
+ ignore_result=True,
+ queue=CeleryQueue.FEATURE_FLAGS.value,
+ autoretry_for=(Exception,),
+ retry_backoff=True,
+ max_retries=5,
+)
@skip_team_scope_audit
def clear_team_definitions_cache(team_id: int) -> None:
- """Clear the /flags/definitions local-eval cache for a specific team, enqueued by staff tooling."""
- clear_flag_definition_caches(team_id)
+ """Clear the /flags/definitions local-eval cache for a team, on team deletion or from staff tooling."""
+ # strict_mirror raises on a failed delete of the shared Redis copy too, so the task retries it.
+ flag_definitions_hypercache.delete_cache_entry(team_id, strict_mirror=True)
@shared_task(bind=True, base=PushGatewayTask, ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS_LONG_RUNNING.value)
diff --git a/products/feature_flags/backend/test/test_local_evaluation.py b/products/feature_flags/backend/test/test_local_evaluation.py
index 0b5b1c31d5b6..686052738642 100644
--- a/products/feature_flags/backend/test/test_local_evaluation.py
+++ b/products/feature_flags/backend/test/test_local_evaluation.py
@@ -16,6 +16,7 @@ from posthog.models.project import Project
from posthog.models.tag import Tag
from posthog.models.team.team import Team
from posthog.personhog_client.fake_client import get_active_fake
+from posthog.storage import object_storage
from posthog.test.persons import _seed_group_type_mapping_into_fake, create_group_type_mapping
from posthog.test.test_utils import create_group_type_mapping_without_created_at
from posthog.utils import safe_cache_delete
@@ -412,6 +413,24 @@ class TestUpdateFlagCachesGroupMappingGuards(BaseTest):
class TestLocalEvaluationSignals(BaseTest):
+ def test_team_delete_clears_definitions_cache_in_redis_and_s3(self):
+ FeatureFlag.objects.create(
+ team=self.team,
+ key="test-flag",
+ created_by=self.user,
+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},
+ )
+ update_flag_definitions_cache(self.team)
+ cache_key = flag_definitions_hypercache.get_cache_key(self.team)
+ assert flag_definitions_hypercache.cache_client.get(cache_key) is not None
+ assert object_storage.read(cache_key, missing_ok=True) is not None
+
+ with self.captureOnCommitCallbacks(execute=True):
+ self.team.delete()
+
+ assert flag_definitions_hypercache.cache_client.get(cache_key) is None
+ assert object_storage.read(cache_key, missing_ok=True) is None
+
@parameterized.expand(["create", "soft_delete", "delete"])
@patch("products.feature_flags.backend.tasks.update_team_flags_cache")
@patch("django.db.transaction.on_commit", lambda fn: fn())
diff --git a/products/feature_flags/backend/test/test_staff_cache_api.py b/products/feature_flags/backend/test/test_staff_cache_api.py
index 8e89f9cedc97..6aad13e5f131 100644
--- a/products/feature_flags/backend/test/test_staff_cache_api.py
+++ b/products/feature_flags/backend/test/test_staff_cache_api.py
@@ -109,20 +109,23 @@ class TestFeatureFlagsStaffCacheAPI(APIBaseTest):
mock_definitions.delay.assert_not_called()
@parameterized.expand(MUTATION_CASES)
- def test_mutation_reports_unknown_team_ids(
- self, _name, url, evaluation_target, definitions_task, assert_evaluation_called
- ):
+ def test_mutation_reports_unknown_team_ids(self, name, url, evaluation_target, definitions_task, _assert_called):
missing_id = self.team.id + 9999
+ expected_queued = [self.team.id, missing_id] if name == "clear" else [self.team.id]
with (
patch(f"products.feature_flags.backend.api.staff_cache.{evaluation_target}") as mock_evaluation,
- patch(f"products.feature_flags.backend.api.staff_cache.{definitions_task}"),
+ patch(f"products.feature_flags.backend.api.staff_cache.{definitions_task}") as mock_definitions,
):
response = self.client.post(url, {"team_ids": [self.team.id, missing_id]}, format="json")
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
- self.assertEqual(response.json()["queued_team_ids"], [self.team.id])
+ self.assertEqual(response.json()["queued_team_ids"], expected_queued)
self.assertEqual(response.json()["not_found_team_ids"], [missing_id])
- assert_evaluation_called(mock_evaluation, self.team.id)
+ evaluation_calls = (
+ mock_evaluation.call_args_list if name == "rebuild" else mock_evaluation.delay.call_args_list
+ )
+ self.assertEqual([call.args[0] for call in evaluation_calls], expected_queued)
+ self.assertEqual([call.args[0] for call in mock_definitions.delay.call_args_list], expected_queued)
def test_rebuild_over_max_team_ids_returns_400(self):
team_ids = list(range(1, MAX_TEAMS_PER_MUTATION + 2))