:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): add command to sweep orphaned team metadata cache entries (#129389)
Phil Haack6c904352eb02
diff --git a/posthog/management/commands/sweep_orphaned_team_metadata_cache.py b/posthog/management/commands/sweep_orphaned_team_metadata_cache.pynew file mode 100644index 000000000000..64c1457b9dbb--- /dev/null+++ b/posthog/management/commands/sweep_orphaned_team_metadata_cache.py@@ -0,0 +1,88 @@+"""+Management command to find and delete team metadata cache entries whose token no team has.++The flags service trusts a team metadata cache hit without a Postgres check, and the+object-storage copy has no TTL. An entry left behind by a deleted team, or by the old+token of a reset team, keeps resolving until something deletes it.++IMPORTANT: This command requires FLAGS_REDIS_URL to be set, so the delete reaches the+Redis copy that the flags service reads.++Usage:+ # List orphaned entries without changing anything+ python manage.py sweep_orphaned_team_metadata_cache++ # Delete orphaned entries from Redis, object storage, and expiry tracking+ python manage.py sweep_orphaned_team_metadata_cache --delete++ # Delete at most 10 orphaned entries+ python manage.py sweep_orphaned_team_metadata_cache --delete --limit 10+"""++from django.core.management.base import CommandError, CommandParser++from posthog.management.commands._base_hypercache_command import BaseHyperCacheCommand+from posthog.models.utils import mask_key_value+from posthog.storage.hypercache_manager import HyperCacheManagementConfig, sweep_orphaned_token_entries+from posthog.storage.team_metadata_cache import TEAM_HYPERCACHE_MANAGEMENT_CONFIG+++class Command(BaseHyperCacheCommand):+ help = "Find team metadata cache entries whose token no team has, and delete them with --delete"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument(+ "--delete",+ action="store_true",+ help="Delete the orphaned entries. Without this flag the command only lists them.",+ )+ parser.add_argument(+ "--limit",+ type=int,+ help="Stop after this many orphaned entries",+ )+ parser.add_argument(+ "--batch-size",+ type=int,+ default=1000,+ help="Number of tokens to check against Postgres per query (default: 1000)",+ )++ def get_hypercache_config(self) -> HyperCacheManagementConfig:+ return TEAM_HYPERCACHE_MANAGEMENT_CONFIG++ def handle(self, *args, **options) -> None:+ if not self.check_dedicated_cache_configured():+ return++ delete: bool = options["delete"]+ limit: int | None = options["limit"]+ batch_size: int = options["batch_size"]++ if limit is not None and limit < 1:+ raise CommandError("--limit must be at least 1")+ if not self.validate_batch_size(batch_size):+ return++ self.print_header("Sweep orphaned team metadata cache entries" + ("" if delete else " (dry run)"))+ result = sweep_orphaned_token_entries(+ self.get_hypercache_config().hypercache,+ delete=delete,+ limit=limit,+ batch_size=batch_size,+ )++ self.stdout.write(f"Tokens checked: {result.tokens_checked}")+ self.stdout.write(f"Orphaned entries: {len(result.orphaned_tokens)}")+ for token in result.orphaned_tokens:+ self.stdout.write(f" {mask_key_value(token)}")++ if not delete:+ if result.orphaned_tokens:+ self.stdout.write(self.style.WARNING("Dry run: nothing was deleted. Re-run with --delete to delete."))+ return++ self.stdout.write(self.style.SUCCESS(f"Deleted: {len(result.orphaned_tokens) - len(result.failed_tokens)}"))+ if result.failed_tokens:+ masked = ", ".join(mask_key_value(token) for token in result.failed_tokens)+ raise CommandError(f"Failed to delete {len(result.failed_tokens)} entries: {masked}")diff --git a/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.py b/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.pynew file mode 100644index 000000000000..c79f90dc6e61--- /dev/null+++ b/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.py@@ -0,0 +1,135 @@+import re+import dataclasses+from io import StringIO+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest+from unittest.mock import patch++from django.core.management import call_command+from django.core.management.base import CommandError+from django.db import OperationalError+from django.test import override_settings++from parameterized import parameterized++from posthog.caching.flags_redis_cache import FLAGS_DEDICATED_CACHE_ALIAS+from posthog.models.utils import mask_key_value+from posthog.storage import object_storage+from posthog.storage.hypercache import HyperCache, HyperCacheStoreMissing+from posthog.storage.team_metadata_cache import TEAM_HYPERCACHE_MANAGEMENT_CONFIG++COMMAND_MODULE = "posthog.management.commands.sweep_orphaned_team_metadata_cache"+++@override_settings(+ OBJECT_STORAGE_ENABLED=True,+ CACHES={+ "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},+ FLAGS_DEDICATED_CACHE_ALIAS: {+ "BACKEND": "django.core.cache.backends.locmem.LocMemCache",+ "LOCATION": "redis://stub:6379/",+ },+ },+)+class TestSweepOrphanedTeamMetadataCache(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ # A namespace of its own keeps the sweep away from entries other tests write to the shared bucket.+ self.hypercache = HyperCache(+ namespace=f"sweep_test_{uuid4().hex}",+ value="full_metadata.json",+ token_based=True,+ load_fn=lambda key: HyperCacheStoreMissing(),+ )+ config_patch = patch(+ f"{COMMAND_MODULE}.TEAM_HYPERCACHE_MANAGEMENT_CONFIG",+ dataclasses.replace(TEAM_HYPERCACHE_MANAGEMENT_CONFIG, hypercache=self.hypercache),+ )+ config_patch.start()+ self.addCleanup(config_patch.stop)++ self.hypercache.set_cache_value(self.team, {"id": self.team.id})+ self.addCleanup(self.hypercache.delete_cache_entry, self.team.api_token)++ def _write_orphan(self, prefix: str = "phc_orphan_") -> str:+ token = f"{prefix}{uuid4().hex}"+ self.hypercache.set_cache_value(token, {"id": 999999})+ self.addCleanup(self.hypercache.delete_cache_entry, token)+ return token++ def _entry_exists(self, token: str) -> bool:+ key = self.hypercache.get_cache_key(token)+ in_s3 = object_storage.read(key, missing_ok=True) is not None+ in_redis = self.hypercache.cache_client.get(key) is not None+ assert in_s3 == in_redis+ return in_s3++ def _run(self, *args: str) -> str:+ out = StringIO()+ call_command("sweep_orphaned_team_metadata_cache", *args, stdout=out)+ return out.getvalue()++ @parameterized.expand([("dry_run", [], True), ("delete", ["--delete"], False)])+ def test_sweep_reports_orphans_and_deletes_only_with_flag(+ self, _name: str, args: list[str], orphan_survives: bool+ ) -> None:+ orphans = [self._write_orphan(), self._write_orphan(prefix="phc_custom/")]+ sibling_cache = HyperCache(+ namespace=self.hypercache.namespace,+ value="other.json",+ token_based=True,+ load_fn=lambda key: HyperCacheStoreMissing(),+ )+ sibling_only_token = f"phc_sibling_{uuid4().hex}"+ sibling_cache.set_cache_value(sibling_only_token, {"id": 999999})+ self.addCleanup(sibling_cache.delete_cache_entry, sibling_only_token)++ output = self._run(*args)++ for orphan in orphans:+ assert mask_key_value(orphan) in output+ assert orphan not in output+ assert self._entry_exists(orphan) is orphan_survives+ assert self.team.api_token not in output+ assert mask_key_value(sibling_only_token) not in output+ assert self._entry_exists(self.team.api_token)+ assert object_storage.read(sibling_cache.get_cache_key(sibling_only_token), missing_ok=True) is not None++ def test_limit_stops_after_n_orphans_across_batches(self) -> None:+ orphans = [self._write_orphan() for _ in range(3)]++ self._run("--delete", "--limit", "2", "--batch-size", "2")++ assert [self._entry_exists(token) for token in orphans].count(False) == 2+ assert self._entry_exists(self.team.api_token)++ def test_team_lookup_error_in_a_later_batch_deletes_nothing(self) -> None:+ orphans = [self._write_orphan(), self._write_orphan()]++ with (+ patch(+ "posthog.storage.hypercache_manager._existing_api_tokens",+ side_effect=[set(), OperationalError("down")],+ ),+ pytest.raises(OperationalError),+ ):+ self._run("--delete", "--batch-size", "1")++ assert all(self._entry_exists(token) for token in orphans)++ def test_redis_delete_failure_keeps_the_object_for_the_next_sweep(self) -> None:+ orphan = self._write_orphan()++ with (+ patch.object(self.hypercache.cache_client, "delete", side_effect=ConnectionError("down")),+ pytest.raises(CommandError, match=re.escape(mask_key_value(orphan))),+ ):+ self._run("--delete")++ assert self._entry_exists(orphan)++ self._run("--delete")++ assert not self._entry_exists(orphan)diff --git a/posthog/storage/hypercache.py b/posthog/storage/hypercache.pyindex d79be7e2772f..5940428d8987 100644--- a/posthog/storage/hypercache.py+++ b/posthog/storage/hypercache.py@@ -1,7 +1,7 @@import jsonimport timeimport hashlib-from collections.abc import Callable+from collections.abc import Callable, Iteratorfrom typing import Optionalfrom django.conf import settings@@ -33,6 +33,8 @@ _REDIS_READ_ERRORS = ()+TOKEN_CACHE_KEY_PREFIX = "cache/team_tokens/"+DEFAULT_CACHE_MISS_TTL = 60 * 60 * 24 # 1 day - it will be invalidated by the daily syncDEFAULT_CACHE_TTL = 60 * 60 * 24 * 30 # 30 days@@ -249,12 +251,29 @@ class HyperCache:if self.token_based:if isinstance(key, Team):key = key.api_token- return f"cache/team_tokens/{key}/{self.namespace}/{self.value}"+ return f"{TOKEN_CACHE_KEY_PREFIX}{key}/{self.namespace}/{self.value}"else:if isinstance(key, Team):key = key.idreturn f"cache/teams/{key}/{self.namespace}/{self.value}"+ def iter_object_storage_tokens(self) -> Iterator[str]:+ """Yield the token of each object-storage entry of this token-based cache.++ All token-based caches share one key prefix. The listing covers that prefix and+ skips the keys of the other caches. Raises ObjectStorageError when the listing fails.+ """+ if not self.token_based:+ raise ValueError("iter_object_storage_tokens requires a token-based cache")+ # A custom token can contain "/", so the token is the text between the prefix and this cache's suffix.+ suffix = f"/{self.namespace}/{self.value}"+ for key in object_storage.iter_keys(TOKEN_CACHE_KEY_PREFIX):+ if not key.endswith(suffix):+ continue+ token = key.removeprefix(TOKEN_CACHE_KEY_PREFIX).removesuffix(suffix)+ if token and self.get_cache_key(token) == key:+ yield token+def get_etag_key(self, key: KeyType) -> str:return f"{self.get_cache_key(key)}:etag"diff --git a/posthog/storage/hypercache_manager.py b/posthog/storage/hypercache_manager.pyindex 2c9cf9b9e3bc..5f4bdf7b29fa 100644--- a/posthog/storage/hypercache_manager.py+++ b/posthog/storage/hypercache_manager.py@@ -13,13 +13,14 @@ Operations include:import randomimport statisticsfrom collections.abc import Callable+from itertools import batchedfrom typing import TYPE_CHECKING, Any, Protocolif TYPE_CHECKING:from django.db.models import QuerySetfrom django.conf import settings-from django.db import connection+from django.db import DEFAULT_DB_ALIAS, connectionimport structlogfrom posthoganalytics import capture_exception@@ -28,6 +29,7 @@ from prometheus_client import Counter, Gaugefrom posthog.dataclasses import frozenfrom posthog.metrics import pushed_metrics_registryfrom posthog.models.team.team import Team+from posthog.models.utils import mask_key_valuefrom posthog.redis import get_clientfrom posthog.storage.hypercache import HyperCache@@ -732,3 +734,64 @@ def batch_check_expiry_tracking(results[identifier] = score is not Nonereturn results+++@frozen+class OrphanedTokenSweepResult:+ tokens_checked: int+ orphaned_tokens: tuple[str, ...]+ failed_tokens: tuple[str, ...]+++def _existing_api_tokens(tokens: tuple[str, ...]) -> set[str]:+ # The primary avoids replica lag, which would report a team created moments ago as missing.+ return set(Team.objects.using(DEFAULT_DB_ALIAS).filter(api_token__in=tokens).values_list("api_token", flat=True))+++def sweep_orphaned_token_entries(+ hypercache: HyperCache,+ *,+ delete: bool,+ limit: int | None = None,+ batch_size: int = 1000,+) -> OrphanedTokenSweepResult:+ """Find the object-storage entries of a token-based cache whose token no team has, and optionally delete them.++ An entry is orphaned only when Postgres has no Team row with its token. The payload is never read.+ The sweep checks every listed token before it deletes any entry.+ A failed listing or Team lookup raises, so the sweep deletes nothing.+ A failed delete is logged and recorded in failed_tokens, and the sweep continues.+ The sweep stops after `limit` orphaned entries.+ """+ tokens_checked = 0+ orphaned: list[str] = []+ for batch in batched(hypercache.iter_object_storage_tokens(), batch_size, strict=False):+ tokens_checked += len(batch)+ existing = _existing_api_tokens(batch)+ orphaned.extend(token for token in batch if token not in existing)+ if limit is not None and len(orphaned) >= limit:+ del orphaned[limit:]+ break++ failed: list[str] = []+ if delete:+ for token in orphaned:+ masked_token = mask_key_value(token)+ try:+ # The sweep finds entries only by listing object storage. The object goes last, so a+ # failed Redis delete leaves it in place and the next sweep finds the token again.+ hypercache.delete_cache_entry(token, kinds=["redis"], strict_mirror=True)+ hypercache.delete_cache_entry(token, kinds=["s3"])+ logger.info("Deleted orphaned cache entry", namespace=hypercache.namespace, token=masked_token)+ except Exception as e:+ logger.exception(+ "Failed to delete orphaned cache entry", namespace=hypercache.namespace, token=masked_token+ )+ capture_exception(e)+ failed.append(token)++ return OrphanedTokenSweepResult(+ tokens_checked=tokens_checked,+ orphaned_tokens=tuple(orphaned),+ failed_tokens=tuple(failed),+ )diff --git a/posthog/storage/object_storage.py b/posthog/storage/object_storage.pyindex af38295b7c11..e8a919b5ab5c 100644--- a/posthog/storage/object_storage.py+++ b/posthog/storage/object_storage.py@@ -1,5 +1,6 @@import abcimport threading+from collections.abc import Iteratorfrom typing import IO, Any, Optional, Unionfrom urllib.parse import urlparse@@ -59,6 +60,11 @@ class ObjectStorageClient(metaclass=abc.ABCMeta):def list_objects(self, bucket: str, prefix: str) -> Optional[list[str]]:pass+ @abc.abstractmethod+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:+ """Yield every key under the prefix, one listing page at a time. Raises ObjectStorageError on any failure."""+ pass+@abc.abstractmethoddef read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:pass@@ -140,6 +146,10 @@ class UnavailableStorage(ObjectStorageClient):def list_objects(self, bucket: str, prefix: str) -> Optional[list[str]]:pass+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:+ # This raises instead of yielding nothing, because an empty listing tells the caller that the prefix holds no keys.+ raise ObjectStorageError("object storage is unavailable")+def read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:return None@@ -287,6 +297,19 @@ class ObjectStorage(ObjectStorageClient):capture_exception(e)return None+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:+ pages = self.aws_client.get_paginator("list_objects_v2").paginate(+ Bucket=bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size}+ )+ try:+ for page in pages:+ for obj in page.get("Contents", []):+ yield obj["Key"]+ except Exception as e:+ logger.exception("object_storage.iter_keys_failed", bucket=bucket, prefix=prefix, error=e)+ capture_exception(e)+ raise ObjectStorageError("list failed") from e+def read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:object_bytes = self.read_bytes(bucket, key, missing_ok=missing_ok)if object_bytes:@@ -597,6 +620,12 @@ def list_objects(prefix: str, bucket: str | None = None) -> Optional[list[str]]:return object_storage_client().list_objects(bucket=bucket or settings.OBJECT_STORAGE_BUCKET, prefix=prefix)+def iter_keys(prefix: str, bucket: str | None = None, *, page_size: int = 1000) -> Iterator[str]:+ return object_storage_client().iter_keys(+ bucket=bucket or settings.OBJECT_STORAGE_BUCKET, prefix=prefix, page_size=page_size+ )++def copy_objects(source_prefix: str, target_prefix: str) -> int:return (object_storage_client().copy_objects(diff --git a/posthog/storage/test/test_object_storage.py b/posthog/storage/test/test_object_storage.pyindex 32b500619c9f..ea6fae63b1bb 100644--- a/posthog/storage/test/test_object_storage.py+++ b/posthog/storage/test/test_object_storage.py@@ -30,6 +30,7 @@ from posthog.storage.object_storage import (get_presigned_url,health_check,is_usable_endpoint,+ iter_keys,list_objects,object_storage_client,read,@@ -164,6 +165,7 @@ class TestStorage(APIBaseTest):"test_storage_bucket/a_shared_prefix/b","test_storage_bucket/a_shared_prefix/c",]+ assert list(iter_keys(prefix=f"{TEST_BUCKET}/{shared_prefix}", page_size=2)) == listing@patch("posthog.storage.object_storage.capture_exception")def test_can_list_unknown_prefix(self, mock_capture_exception) -> None: