Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): clear team metadata cache when a team is deleted (#129217)

Phil Haacka2605f7404ae
diff --git a/docs/internal/feature-flags/hypercache-system.md b/docs/internal/feature-flags/hypercache-system.md
index be7da8e895a7..67ef2f3876c6 100644
--- a/docs/internal/feature-flags/hypercache-system.md
+++ b/docs/internal/feature-flags/hypercache-system.md
@@ -234,11 +234,13 @@ Models that invalidate the flags cache:
- `FeatureFlagEvaluationTag` - Evaluation tags changed
- `Tag` - Tag renamed
-Deleting a `Team` clears its local evaluation entry from Redis and S3 after the deletion commits.
-The rebuild task cannot run for a deleted team, and `/flags/definitions` falls back to the S3 copy, which has no TTL.
+Deleting a `Team` clears its local evaluation and team metadata entries from Redis and S3 after the deletion commits.
+The rebuild task cannot run for a deleted team, and the flags service falls back to the S3 copies, which have no TTL.
+A stale team metadata entry lets the flags service resolve the deleted team's token without a Postgres check.
The staff cache tools `clear` action also accepts the ids of deleted teams.
-Use it to purge an orphaned entry for a team that no longer exists.
+Use it to purge an orphaned flags entry for a team that no longer exists.
+It does not clear team metadata entries, because it takes team ids and those entries are keyed by API token.
## Remote config caching
diff --git a/posthog/storage/team_metadata_cache.py b/posthog/storage/team_metadata_cache.py
index ff292710e88e..3f74e6f2ebb2 100644
--- a/posthog/storage/team_metadata_cache.py
+++ b/posthog/storage/team_metadata_cache.py
@@ -30,7 +30,7 @@ Cache Invalidation:
-------------------
Caches are invalidated automatically when:
- Team is saved (via Django signal → Celery task)
-- Team is deleted (via Django signal → immediate clear)
+- Team is deleted (via Django signal → retrying Celery task that clears the entry by token after commit)
- Hourly refresh job detects expiring entries (TTL < 24h)
Manual invalidation:
diff --git a/posthog/storage/team_metadata_signal_handlers.py b/posthog/storage/team_metadata_signal_handlers.py
index be5fd134ab2b..5bb67e9ac588 100644
--- a/posthog/storage/team_metadata_signal_handlers.py
+++ b/posthog/storage/team_metadata_signal_handlers.py
@@ -9,6 +9,7 @@ that has not sent a task yet would leave the flags service reading stale team me
until the hourly verification fixes it.
"""
+from collections.abc import Callable
from typing import Any
from django.conf import settings
@@ -34,21 +35,25 @@ def _record_enqueue_failure() -> None:
).inc()
+def _enqueue_or_record_failure(enqueue: Callable[[], object], failure_message: str, **log_context: Any) -> None:
+ try:
+ enqueue()
+ except Exception as e:
+ _record_enqueue_failure()
+ logger.exception(failure_message, error=str(e), **log_context)
+
+
def _enqueue_team_metadata_update_on_commit(team_id: int) -> None:
def enqueue_task() -> None:
# posthog.tasks.__init__ eagerly imports every task module (celery autoimport);
# this signal module is wired at django.setup(), so import the task lazily.
from posthog.tasks.team_metadata import update_team_metadata_cache_task # noqa: PLC0415
- try:
- update_team_metadata_cache_task.delay(team_id)
- except Exception as e:
- _record_enqueue_failure()
- logger.exception(
- "Failed to enqueue cache update task",
- team_id=team_id,
- error=str(e),
- )
+ _enqueue_or_record_failure(
+ lambda: update_team_metadata_cache_task.delay(team_id),
+ "Failed to enqueue cache update task",
+ team_id=team_id,
+ )
# Use transaction.on_commit to ensure the database changes are committed
# before we update the cache
@@ -59,16 +64,12 @@ def _enqueue_related_team_metadata_fanout(*, organization_id: int | None = None,
# Imported lazily for the same reason as in _enqueue_team_metadata_update_on_commit.
from posthog.tasks.team_metadata import update_related_teams_metadata_cache_task # noqa: PLC0415
- try:
- update_related_teams_metadata_cache_task.delay(organization_id=organization_id, project_id=project_id)
- except Exception as e:
- _record_enqueue_failure()
- logger.exception(
- "Failed to enqueue related team metadata fan-out",
- organization_id=organization_id,
- project_id=project_id,
- error=str(e),
- )
+ _enqueue_or_record_failure(
+ lambda: update_related_teams_metadata_cache_task.delay(organization_id=organization_id, project_id=project_id),
+ "Failed to enqueue related team metadata fan-out",
+ organization_id=organization_id,
+ project_id=project_id,
+ )
def _any_field_may_have_changed(update_fields: frozenset[str] | None, fields: frozenset[str]) -> bool:
@@ -103,6 +104,22 @@ def update_team_metadata_cache_on_flag_policy_save(
_enqueue_team_metadata_update_on_commit(instance.team_id)
+def _enqueue_deleted_team_metadata_clear_on_commit(team_id: int, api_token: str) -> None:
+ # The pre_delete clear runs before the commit and does not retry. A refresh in that gap can write the
+ # entry back. The S3 copy has no TTL, so a retrying task clears the entry again after the commit.
+ def enqueue_task() -> None:
+ # Imported lazily for the same reason as in _enqueue_team_metadata_update_on_commit.
+ from posthog.tasks.team_metadata import clear_team_metadata_cache_for_token_task # noqa: PLC0415
+
+ _enqueue_or_record_failure(
+ lambda: clear_team_metadata_cache_for_token_task.delay(api_token),
+ "Failed to enqueue deleted team metadata clear",
+ team_id=team_id,
+ )
+
+ transaction.on_commit(enqueue_task)
+
+
def clear_team_metadata_cache_on_delete(sender: type[Team], instance: Team, **kwargs: Any) -> None:
"""Clear team metadata cache when a Team is deleted."""
if not settings.FLAGS_REDIS_URL:
@@ -111,6 +128,7 @@ def clear_team_metadata_cache_on_delete(sender: type[Team], instance: Team, **kw
# NB: For unit tests, only clear Redis to avoid S3 timestamp issues with frozen time
kinds = ["redis"] if settings.TEST else None
clear_team_metadata_cache(instance, kinds=kinds)
+ _enqueue_deleted_team_metadata_clear_on_commit(instance.id, instance.api_token)
# Organization and project names are denormalized into team metadata, but only the Team signal
diff --git a/posthog/storage/test/test_team_metadata_signal_handlers.py b/posthog/storage/test/test_team_metadata_signal_handlers.py
index b2c98ddafafd..a770919f8ed4 100644
--- a/posthog/storage/test/test_team_metadata_signal_handlers.py
+++ b/posthog/storage/test/test_team_metadata_signal_handlers.py
@@ -7,6 +7,8 @@ from parameterized import parameterized
from posthog.models.organization import Organization
from posthog.models.project import Project
+from posthog.storage import object_storage
+from posthog.storage.team_metadata_cache import team_metadata_hypercache, update_team_metadata_cache
from products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfig
@@ -104,6 +106,21 @@ class TestOrgProjectDeleteCascadesToTeamClear(BaseTest):
self.assertIn(team_id, cleared_team_ids)
+ def test_team_delete_clears_team_metadata_cache_in_redis_and_s3(self) -> None:
+ update_team_metadata_cache(self.team)
+ cache_key = team_metadata_hypercache.get_cache_key(self.team)
+ assert team_metadata_hypercache.cache_client.get(cache_key) is not None
+ assert object_storage.read(cache_key, missing_ok=True) is not None
+
+ with self.captureOnCommitCallbacks(execute=True):
+ api_token = self.team.api_token
+ self.team.delete()
+ # This write stands in for a refresh that runs after the pre_delete clear and before the commit.
+ team_metadata_hypercache.set_cache_value(api_token, {"api_token": api_token})
+
+ assert team_metadata_hypercache.cache_client.get(cache_key) is None
+ assert object_storage.read(cache_key, missing_ok=True) is None
+
@override_settings(FLAGS_REDIS_URL="redis://localhost:6379")
class TestFlagPolicyBlocklistCacheRefresh(BaseTest):
diff --git a/posthog/tasks/team_metadata.py b/posthog/tasks/team_metadata.py
index a48a71df0a33..07317e52adbe 100644
--- a/posthog/tasks/team_metadata.py
+++ b/posthog/tasks/team_metadata.py
@@ -18,6 +18,7 @@ from posthog.storage.team_metadata_cache import (
cleanup_stale_expiry_tracking,
get_cache_stats,
refresh_expiring_caches,
+ team_metadata_hypercache,
update_team_metadata_cache,
)
from posthog.tasks.utils import CeleryQueue
@@ -85,6 +86,25 @@ def update_related_teams_metadata_cache_task(organization_id: int | None = None,
raise
+# The task retries because no rebuild repairs an entry whose token no longer belongs to a team.
+@shared_task(
+ ignore_result=True,
+ queue=CeleryQueue.DEFAULT.value,
+ autoretry_for=(Exception,),
+ retry_backoff=True,
+ retry_backoff_max=900,
+ max_retries=10,
+)
+@skip_team_scope_audit
+def clear_team_metadata_cache_for_token_task(api_token: str) -> None:
+ """Clear the metadata cache entry for a token that no longer belongs to a team, from Redis and S3.
+
+ The task takes the token because the Team row may be gone, and then no query can find the token from a team id.
+ """
+ # strict_mirror raises on a failed delete of the shared Redis copy too, so the task retries it.
+ team_metadata_hypercache.delete_cache_entry(api_token, strict_mirror=True)
+
+
@shared_task(ignore_result=True, queue=CeleryQueue.DEFAULT.value)
def refresh_expiring_team_metadata_cache_entries() -> None:
"""
diff --git a/posthog/tasks/test/__snapshots__/test_task_registration.ambr b/posthog/tasks/test/__snapshots__/test_task_registration.ambr
index 4cf192e3e691..7b0637069c46 100644
--- a/posthog/tasks/test/__snapshots__/test_task_registration.ambr
+++ b/posthog/tasks/test/__snapshots__/test_task_registration.ambr
@@ -164,6 +164,7 @@
'posthog.tasks.team_llm_gateway_quota.project_teams_llm_gateway_quota_task',
'posthog.tasks.team_llm_gateway_quota.reconcile_llm_gateway_quota_projection',
'posthog.tasks.team_metadata.cleanup_stale_expiry_tracking_task',
+ 'posthog.tasks.team_metadata.clear_team_metadata_cache_for_token_task',
'posthog.tasks.team_metadata.refresh_expiring_team_metadata_cache_entries',
'posthog.tasks.team_metadata.update_related_teams_metadata_cache_task',
'posthog.tasks.team_metadata.update_team_metadata_cache_task',
diff --git a/posthog/tasks/test/test_team_metadata.py b/posthog/tasks/test/test_team_metadata.py
index 7e2e78141ccf..a38fc983240a 100644
--- a/posthog/tasks/test/test_team_metadata.py
+++ b/posthog/tasks/test/test_team_metadata.py
@@ -1,10 +1,15 @@
from posthog.test.base import BaseTest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
from django.test import override_settings
from posthog.models.project import Project
-from posthog.tasks.team_metadata import update_related_teams_metadata_cache_task
+from posthog.storage import object_storage
+from posthog.tasks.team_metadata import (
+ clear_team_metadata_cache_for_token_task,
+ team_metadata_hypercache,
+ update_related_teams_metadata_cache_task,
+)
@override_settings(FLAGS_REDIS_URL="redis://localhost:6379")
@@ -43,3 +48,16 @@ class TestRelatedTeamsMetadataFanoutTask(BaseTest):
update_related_teams_metadata_cache_task(organization_id=self.organization.id)
mock_logger.exception.assert_called_once()
+
+
+class TestClearTeamMetadataCacheForTokenTask(BaseTest):
+ def test_failed_mirror_delete_raises_so_the_task_retries(self) -> None:
+ mirror = Mock()
+ mirror.delete.side_effect = ConnectionError("mirror down")
+
+ with (
+ patch.object(team_metadata_hypercache, "secondary_cache_client", mirror),
+ patch.object(object_storage, "delete"),
+ self.assertRaises(ConnectionError),
+ ):
+ clear_team_metadata_cache_for_token_task(self.team.api_token)