:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): clear team metadata cache when a team is deleted (#129217)
Phil Haacka2605f7404ae
diff --git a/docs/internal/feature-flags/hypercache-system.md b/docs/internal/feature-flags/hypercache-system.mdindex be7da8e895a7..67ef2f3876c6 100644--- a/docs/internal/feature-flags/hypercache-system.md+++ b/docs/internal/feature-flags/hypercache-system.md@@ -234,11 +234,13 @@ Models that invalidate the flags cache:- `FeatureFlagEvaluationTag` - Evaluation tags changed- `Tag` - Tag renamed-Deleting a `Team` clears its local evaluation entry from Redis and S3 after the deletion commits.-The rebuild task cannot run for a deleted team, and `/flags/definitions` falls back to the S3 copy, which has no TTL.+Deleting a `Team` clears its local evaluation and team metadata entries from Redis and S3 after the deletion commits.+The rebuild task cannot run for a deleted team, and the flags service falls back to the S3 copies, which have no TTL.+A stale team metadata entry lets the flags service resolve the deleted team's token without a Postgres check.The staff cache tools `clear` action also accepts the ids of deleted teams.-Use it to purge an orphaned entry for a team that no longer exists.+Use it to purge an orphaned flags entry for a team that no longer exists.+It does not clear team metadata entries, because it takes team ids and those entries are keyed by API token.## Remote config cachingdiff --git a/posthog/storage/team_metadata_cache.py b/posthog/storage/team_metadata_cache.pyindex ff292710e88e..3f74e6f2ebb2 100644--- a/posthog/storage/team_metadata_cache.py+++ b/posthog/storage/team_metadata_cache.py@@ -30,7 +30,7 @@ Cache Invalidation:-------------------Caches are invalidated automatically when:- Team is saved (via Django signal → Celery task)-- Team is deleted (via Django signal → immediate clear)+- Team is deleted (via Django signal → retrying Celery task that clears the entry by token after commit)- Hourly refresh job detects expiring entries (TTL < 24h)Manual invalidation:diff --git a/posthog/storage/team_metadata_signal_handlers.py b/posthog/storage/team_metadata_signal_handlers.pyindex be5fd134ab2b..5bb67e9ac588 100644--- a/posthog/storage/team_metadata_signal_handlers.py+++ b/posthog/storage/team_metadata_signal_handlers.py@@ -9,6 +9,7 @@ that has not sent a task yet would leave the flags service reading stale team meuntil the hourly verification fixes it."""+from collections.abc import Callablefrom typing import Anyfrom django.conf import settings@@ -34,21 +35,25 @@ def _record_enqueue_failure() -> None:).inc()+def _enqueue_or_record_failure(enqueue: Callable[[], object], failure_message: str, **log_context: Any) -> None:+ try:+ enqueue()+ except Exception as e:+ _record_enqueue_failure()+ logger.exception(failure_message, error=str(e), **log_context)++def _enqueue_team_metadata_update_on_commit(team_id: int) -> None:def enqueue_task() -> None:# posthog.tasks.__init__ eagerly imports every task module (celery autoimport);# this signal module is wired at django.setup(), so import the task lazily.from posthog.tasks.team_metadata import update_team_metadata_cache_task # noqa: PLC0415- try:- update_team_metadata_cache_task.delay(team_id)- except Exception as e:- _record_enqueue_failure()- logger.exception(- "Failed to enqueue cache update task",- team_id=team_id,- error=str(e),- )+ _enqueue_or_record_failure(+ lambda: update_team_metadata_cache_task.delay(team_id),+ "Failed to enqueue cache update task",+ team_id=team_id,+ )# Use transaction.on_commit to ensure the database changes are committed# before we update the cache@@ -59,16 +64,12 @@ def _enqueue_related_team_metadata_fanout(*, organization_id: int | None = None,# Imported lazily for the same reason as in _enqueue_team_metadata_update_on_commit.from posthog.tasks.team_metadata import update_related_teams_metadata_cache_task # noqa: PLC0415- try:- update_related_teams_metadata_cache_task.delay(organization_id=organization_id, project_id=project_id)- except Exception as e:- _record_enqueue_failure()- logger.exception(- "Failed to enqueue related team metadata fan-out",- organization_id=organization_id,- project_id=project_id,- error=str(e),- )+ _enqueue_or_record_failure(+ lambda: update_related_teams_metadata_cache_task.delay(organization_id=organization_id, project_id=project_id),+ "Failed to enqueue related team metadata fan-out",+ organization_id=organization_id,+ project_id=project_id,+ )def _any_field_may_have_changed(update_fields: frozenset[str] | None, fields: frozenset[str]) -> bool:@@ -103,6 +104,22 @@ def update_team_metadata_cache_on_flag_policy_save(_enqueue_team_metadata_update_on_commit(instance.team_id)+def _enqueue_deleted_team_metadata_clear_on_commit(team_id: int, api_token: str) -> None:+ # The pre_delete clear runs before the commit and does not retry. A refresh in that gap can write the+ # entry back. The S3 copy has no TTL, so a retrying task clears the entry again after the commit.+ def enqueue_task() -> None:+ # Imported lazily for the same reason as in _enqueue_team_metadata_update_on_commit.+ from posthog.tasks.team_metadata import clear_team_metadata_cache_for_token_task # noqa: PLC0415++ _enqueue_or_record_failure(+ lambda: clear_team_metadata_cache_for_token_task.delay(api_token),+ "Failed to enqueue deleted team metadata clear",+ team_id=team_id,+ )++ transaction.on_commit(enqueue_task)++def clear_team_metadata_cache_on_delete(sender: type[Team], instance: Team, **kwargs: Any) -> None:"""Clear team metadata cache when a Team is deleted."""if not settings.FLAGS_REDIS_URL:@@ -111,6 +128,7 @@ def clear_team_metadata_cache_on_delete(sender: type[Team], instance: Team, **kw# NB: For unit tests, only clear Redis to avoid S3 timestamp issues with frozen timekinds = ["redis"] if settings.TEST else Noneclear_team_metadata_cache(instance, kinds=kinds)+ _enqueue_deleted_team_metadata_clear_on_commit(instance.id, instance.api_token)# Organization and project names are denormalized into team metadata, but only the Team signaldiff --git a/posthog/storage/test/test_team_metadata_signal_handlers.py b/posthog/storage/test/test_team_metadata_signal_handlers.pyindex b2c98ddafafd..a770919f8ed4 100644--- a/posthog/storage/test/test_team_metadata_signal_handlers.py+++ b/posthog/storage/test/test_team_metadata_signal_handlers.py@@ -7,6 +7,8 @@ from parameterized import parameterizedfrom posthog.models.organization import Organizationfrom posthog.models.project import Project+from posthog.storage import object_storage+from posthog.storage.team_metadata_cache import team_metadata_hypercache, update_team_metadata_cachefrom products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfig@@ -104,6 +106,21 @@ class TestOrgProjectDeleteCascadesToTeamClear(BaseTest):self.assertIn(team_id, cleared_team_ids)+ def test_team_delete_clears_team_metadata_cache_in_redis_and_s3(self) -> None:+ update_team_metadata_cache(self.team)+ cache_key = team_metadata_hypercache.get_cache_key(self.team)+ assert team_metadata_hypercache.cache_client.get(cache_key) is not None+ assert object_storage.read(cache_key, missing_ok=True) is not None++ with self.captureOnCommitCallbacks(execute=True):+ api_token = self.team.api_token+ self.team.delete()+ # This write stands in for a refresh that runs after the pre_delete clear and before the commit.+ team_metadata_hypercache.set_cache_value(api_token, {"api_token": api_token})++ assert team_metadata_hypercache.cache_client.get(cache_key) is None+ assert object_storage.read(cache_key, missing_ok=True) is None+@override_settings(FLAGS_REDIS_URL="redis://localhost:6379")class TestFlagPolicyBlocklistCacheRefresh(BaseTest):diff --git a/posthog/tasks/team_metadata.py b/posthog/tasks/team_metadata.pyindex a48a71df0a33..07317e52adbe 100644--- a/posthog/tasks/team_metadata.py+++ b/posthog/tasks/team_metadata.py@@ -18,6 +18,7 @@ from posthog.storage.team_metadata_cache import (cleanup_stale_expiry_tracking,get_cache_stats,refresh_expiring_caches,+ team_metadata_hypercache,update_team_metadata_cache,)from posthog.tasks.utils import CeleryQueue@@ -85,6 +86,25 @@ def update_related_teams_metadata_cache_task(organization_id: int | None = None,raise+# The task retries because no rebuild repairs an entry whose token no longer belongs to a team.+@shared_task(+ ignore_result=True,+ queue=CeleryQueue.DEFAULT.value,+ autoretry_for=(Exception,),+ retry_backoff=True,+ retry_backoff_max=900,+ max_retries=10,+)+@skip_team_scope_audit+def clear_team_metadata_cache_for_token_task(api_token: str) -> None:+ """Clear the metadata cache entry for a token that no longer belongs to a team, from Redis and S3.++ The task takes the token because the Team row may be gone, and then no query can find the token from a team id.+ """+ # strict_mirror raises on a failed delete of the shared Redis copy too, so the task retries it.+ team_metadata_hypercache.delete_cache_entry(api_token, strict_mirror=True)++@shared_task(ignore_result=True, queue=CeleryQueue.DEFAULT.value)def refresh_expiring_team_metadata_cache_entries() -> None:"""diff --git a/posthog/tasks/test/__snapshots__/test_task_registration.ambr b/posthog/tasks/test/__snapshots__/test_task_registration.ambrindex 4cf192e3e691..7b0637069c46 100644--- a/posthog/tasks/test/__snapshots__/test_task_registration.ambr+++ b/posthog/tasks/test/__snapshots__/test_task_registration.ambr@@ -164,6 +164,7 @@'posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task','posthog.tasks.team_llm_gateway_quota.reconcile_llm_gateway_quota_projection','posthog.tasks.team_metadata.cleanup_stale_expiry_tracking_task',+ 'posthog.tasks.team_metadata.clear_team_metadata_cache_for_token_task','posthog.tasks.team_metadata.refresh_expiring_team_metadata_cache_entries','posthog.tasks.team_metadata.update_related_teams_metadata_cache_task','posthog.tasks.team_metadata.update_team_metadata_cache_task',diff --git a/posthog/tasks/test/test_team_metadata.py b/posthog/tasks/test/test_team_metadata.pyindex 7e2e78141ccf..a38fc983240a 100644--- a/posthog/tasks/test/test_team_metadata.py+++ b/posthog/tasks/test/test_team_metadata.py@@ -1,10 +1,15 @@from posthog.test.base import BaseTest-from unittest.mock import patch+from unittest.mock import Mock, patchfrom django.test import override_settingsfrom posthog.models.project import Project-from posthog.tasks.team_metadata import update_related_teams_metadata_cache_task+from posthog.storage import object_storage+from posthog.tasks.team_metadata import (+ clear_team_metadata_cache_for_token_task,+ team_metadata_hypercache,+ update_related_teams_metadata_cache_task,+)@override_settings(FLAGS_REDIS_URL="redis://localhost:6379")@@ -43,3 +48,16 @@ class TestRelatedTeamsMetadataFanoutTask(BaseTest):update_related_teams_metadata_cache_task(organization_id=self.organization.id)mock_logger.exception.assert_called_once()+++class TestClearTeamMetadataCacheForTokenTask(BaseTest):+ def test_failed_mirror_delete_raises_so_the_task_retries(self) -> None:+ mirror = Mock()+ mirror.delete.side_effect = ConnectionError("mirror down")++ with (+ patch.object(team_metadata_hypercache, "secondary_cache_client", mirror),+ patch.object(object_storage, "delete"),+ self.assertRaises(ConnectionError),+ ):+ clear_team_metadata_cache_for_token_task(self.team.api_token)