:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): clear local evaluation cache when a team is deleted (#128011)
Ben White0c4ff77a5bf2
diff --git a/docs/internal/feature-flags/hypercache-system.md b/docs/internal/feature-flags/hypercache-system.mdindex e21e8bb19593..be7da8e895a7 100644--- a/docs/internal/feature-flags/hypercache-system.md+++ b/docs/internal/feature-flags/hypercache-system.md@@ -234,6 +234,12 @@ Models that invalidate the flags cache:- `FeatureFlagEvaluationTag` - Evaluation tags changed- `Tag` - Tag renamed+Deleting a `Team` clears its local evaluation entry from Redis and S3 after the deletion commits.+The rebuild task cannot run for a deleted team, and `/flags/definitions` falls back to the S3 copy, which has no TTL.++The staff cache tools `clear` action also accepts the ids of deleted teams.+Use it to purge an orphaned entry for a team that no longer exists.+## Remote config cachingRemote config uses a different caching strategy than local evaluation. It prioritizes zero-overhead cache hits by using Redis-only caching for serving, with HyperCache used only for background sync operations.diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txtindex d2947215f27d..1cf302afc5bf 100644--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt@@ -118,6 +118,7 @@ post_delete:products.feature_flags.backend.local_evaluation.cohort_changedpost_delete:products.feature_flags.backend.local_evaluation.evaluation_context_changedpost_delete:products.feature_flags.backend.local_evaluation.experiment_changedpost_delete:products.feature_flags.backend.local_evaluation.feature_flag_changed+post_delete:products.feature_flags.backend.local_evaluation.team_deletedpost_delete:products.managed_warehouse.backend.models._observe_duckgres_server_deletepost_delete:products.replay_vision.backend.media_expiry.expire_media_assetpost_delete:products.review_hog.backend.ownership._invalidate_owned_repositories_on_deletediff --git a/products/feature_flags/backend/api/staff_cache.py b/products/feature_flags/backend/api/staff_cache.pyindex d3ef5d230548..8075c495834f 100644--- a/products/feature_flags/backend/api/staff_cache.py+++ b/products/feature_flags/backend/api/staff_cache.py@@ -312,17 +312,20 @@ def _dispatch_mutation(evaluation_fn: Callable[[int], None],definitions_fn: Callable[[int], None],log_event: str,+ queue_missing_teams: bool = False,) -> response.Response:"""Shared shape of `rebuild` and `clear`: split team_ids into found/not-found, dispatch the- per-cache action for each found team, log, and return the 202 response both actions share."""+ per-cache action for each found team (or every team with `queue_missing_teams`), log, and+ return the 202 response both actions share."""# Dedupe (preserving order) so a caller passing the same id twice doesn't enqueue duplicate# rebuild/clear work for that team.deduped_ids = list(dict.fromkeys(team_ids))found_set = set(Team.objects.filter(id__in=deduped_ids).values_list("id", flat=True))found_ids = [team_id for team_id in deduped_ids if team_id in found_set]not_found_ids = [team_id for team_id in deduped_ids if team_id not in found_set]+ queued_ids = deduped_ids if queue_missing_teams else found_ids- for team_id in found_ids:+ for team_id in queued_ids:if EVALUATION in caches:evaluation_fn(team_id)if DEFINITIONS in caches:@@ -332,13 +335,13 @@ def _dispatch_mutation(log_event,staff_user_id=request.user.id,was_impersonated=is_impersonated(request),- team_ids=found_ids,+ team_ids=queued_ids,not_found_team_ids=not_found_ids,caches=caches,)return response.Response(- {"queued_team_ids": found_ids, "not_found_team_ids": not_found_ids},+ {"queued_team_ids": queued_ids, "not_found_team_ids": not_found_ids},status=status.HTTP_202_ACCEPTED,)@@ -506,4 +509,6 @@ class FeatureFlagsStaffCacheViewSet(viewsets.ViewSet):evaluation_fn=clear_team_evaluation_cache.delay,definitions_fn=clear_team_definitions_cache.delay,log_event="flags_staff_cache_clear",+ # Clearing needs no Team row, so this also purges entries a deleted team left behind.+ queue_missing_teams=True,)diff --git a/products/feature_flags/backend/local_evaluation.py b/products/feature_flags/backend/local_evaluation.pyindex da6fb761fd48..fa6b587c7363 100644--- a/products/feature_flags/backend/local_evaluation.py+++ b/products/feature_flags/backend/local_evaluation.py@@ -988,6 +988,16 @@ def feature_flag_changed(sender, instance: "FeatureFlag", **kwargs):transaction.on_commit(lambda: update_team_flags_cache.delay(instance.team_id))+@receiver(post_delete, sender=Team)+def team_deleted(sender, instance: Team, **kwargs) -> None:+ # The S3 copy has no TTL, and /flags/definitions falls back to it when Redis misses,+ # so without this a deleted team's definitions stay served indefinitely.+ from products.feature_flags.backend.tasks import clear_team_definitions_cache++ team_id = instance.id+ transaction.on_commit(lambda: clear_team_definitions_cache.delay(team_id))++@receiver(post_save, sender=Experiment)@receiver(post_delete, sender=Experiment)def experiment_changed(sender, instance: "Experiment", **kwargs):diff --git a/products/feature_flags/backend/tasks.py b/products/feature_flags/backend/tasks.pyindex ef465c3fe7ea..3a89c1eb76e1 100644--- a/products/feature_flags/backend/tasks.py+++ b/products/feature_flags/backend/tasks.py@@ -35,7 +35,7 @@ from products.feature_flags.backend.flags_cache import ()from products.feature_flags.backend.local_evaluation import (FLAG_DEFINITIONS_HYPERCACHE_MANAGEMENT_CONFIG,- clear_flag_definition_caches,+ flag_definitions_hypercache,update_flag_caches,)from products.feature_flags.backend.models.feature_flag import FeatureFlag@@ -234,11 +234,19 @@ def clear_team_evaluation_cache(team_id: int) -> None:clear_flags_cache(team_id)-@shared_task(ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS.value)+# The task retries because no rebuild can repair the entry once the Team row is gone.+@shared_task(+ ignore_result=True,+ queue=CeleryQueue.FEATURE_FLAGS.value,+ autoretry_for=(Exception,),+ retry_backoff=True,+ max_retries=5,+)@skip_team_scope_auditdef clear_team_definitions_cache(team_id: int) -> None:- """Clear the /flags/definitions local-eval cache for a specific team, enqueued by staff tooling."""- clear_flag_definition_caches(team_id)+ """Clear the /flags/definitions local-eval cache for a team, on team deletion or from staff tooling."""+ # strict_mirror raises on a failed delete of the shared Redis copy too, so the task retries it.+ flag_definitions_hypercache.delete_cache_entry(team_id, strict_mirror=True)@shared_task(bind=True, base=PushGatewayTask, ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS_LONG_RUNNING.value)diff --git a/products/feature_flags/backend/test/test_local_evaluation.py b/products/feature_flags/backend/test/test_local_evaluation.pyindex 0b5b1c31d5b6..686052738642 100644--- a/products/feature_flags/backend/test/test_local_evaluation.py+++ b/products/feature_flags/backend/test/test_local_evaluation.py@@ -16,6 +16,7 @@ from posthog.models.project import Projectfrom posthog.models.tag import Tagfrom posthog.models.team.team import Teamfrom posthog.personhog_client.fake_client import get_active_fake+from posthog.storage import object_storagefrom posthog.test.persons import _seed_group_type_mapping_into_fake, create_group_type_mappingfrom posthog.test.test_utils import create_group_type_mapping_without_created_atfrom posthog.utils import safe_cache_delete@@ -412,6 +413,24 @@ class TestUpdateFlagCachesGroupMappingGuards(BaseTest):class TestLocalEvaluationSignals(BaseTest):+ def test_team_delete_clears_definitions_cache_in_redis_and_s3(self):+ FeatureFlag.objects.create(+ team=self.team,+ key="test-flag",+ created_by=self.user,+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},+ )+ update_flag_definitions_cache(self.team)+ cache_key = flag_definitions_hypercache.get_cache_key(self.team)+ assert flag_definitions_hypercache.cache_client.get(cache_key) is not None+ assert object_storage.read(cache_key, missing_ok=True) is not None++ with self.captureOnCommitCallbacks(execute=True):+ self.team.delete()++ assert flag_definitions_hypercache.cache_client.get(cache_key) is None+ assert object_storage.read(cache_key, missing_ok=True) is None+@parameterized.expand(["create", "soft_delete", "delete"])@patch("products.feature_flags.backend.tasks.update_team_flags_cache")@patch("django.db.transaction.on_commit", lambda fn: fn())diff --git a/products/feature_flags/backend/test/test_staff_cache_api.py b/products/feature_flags/backend/test/test_staff_cache_api.pyindex 8e89f9cedc97..6aad13e5f131 100644--- a/products/feature_flags/backend/test/test_staff_cache_api.py+++ b/products/feature_flags/backend/test/test_staff_cache_api.py@@ -109,20 +109,23 @@ class TestFeatureFlagsStaffCacheAPI(APIBaseTest):mock_definitions.delay.assert_not_called()@parameterized.expand(MUTATION_CASES)- def test_mutation_reports_unknown_team_ids(- self, _name, url, evaluation_target, definitions_task, assert_evaluation_called- ):+ def test_mutation_reports_unknown_team_ids(self, name, url, evaluation_target, definitions_task, _assert_called):missing_id = self.team.id + 9999+ expected_queued = [self.team.id, missing_id] if name == "clear" else [self.team.id]with (patch(f"products.feature_flags.backend.api.staff_cache.{evaluation_target}") as mock_evaluation,- patch(f"products.feature_flags.backend.api.staff_cache.{definitions_task}"),+ patch(f"products.feature_flags.backend.api.staff_cache.{definitions_task}") as mock_definitions,):response = self.client.post(url, {"team_ids": [self.team.id, missing_id]}, format="json")self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)- self.assertEqual(response.json()["queued_team_ids"], [self.team.id])+ self.assertEqual(response.json()["queued_team_ids"], expected_queued)self.assertEqual(response.json()["not_found_team_ids"], [missing_id])- assert_evaluation_called(mock_evaluation, self.team.id)+ evaluation_calls = (+ mock_evaluation.call_args_list if name == "rebuild" else mock_evaluation.delay.call_args_list+ )+ self.assertEqual([call.args[0] for call in evaluation_calls], expected_queued)+ self.assertEqual([call.args[0] for call in mock_definitions.delay.call_args_list], expected_queued)def test_rebuild_over_max_team_ids_returns_400(self):team_ids = list(range(1, MAX_TEAMS_PER_MUTATION + 2))