Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): add command to sweep orphaned team metadata cache entries (#129389)

Phil Haack6c904352eb02
diff --git a/posthog/management/commands/sweep_orphaned_team_metadata_cache.py b/posthog/management/commands/sweep_orphaned_team_metadata_cache.py
new file mode 100644
index 000000000000..64c1457b9dbb
--- /dev/null
+++ b/posthog/management/commands/sweep_orphaned_team_metadata_cache.py
@@ -0,0 +1,88 @@
+"""
+Management command to find and delete team metadata cache entries whose token no team has.
+
+The flags service trusts a team metadata cache hit without a Postgres check, and the
+object-storage copy has no TTL. An entry left behind by a deleted team, or by the old
+token of a reset team, keeps resolving until something deletes it.
+
+IMPORTANT: This command requires FLAGS_REDIS_URL to be set, so the delete reaches the
+Redis copy that the flags service reads.
+
+Usage:
+ # List orphaned entries without changing anything
+ python manage.py sweep_orphaned_team_metadata_cache
+
+ # Delete orphaned entries from Redis, object storage, and expiry tracking
+ python manage.py sweep_orphaned_team_metadata_cache --delete
+
+ # Delete at most 10 orphaned entries
+ python manage.py sweep_orphaned_team_metadata_cache --delete --limit 10
+"""
+
+from django.core.management.base import CommandError, CommandParser
+
+from posthog.management.commands._base_hypercache_command import BaseHyperCacheCommand
+from posthog.models.utils import mask_key_value
+from posthog.storage.hypercache_manager import HyperCacheManagementConfig, sweep_orphaned_token_entries
+from posthog.storage.team_metadata_cache import TEAM_HYPERCACHE_MANAGEMENT_CONFIG
+
+
+class Command(BaseHyperCacheCommand):
+ help = "Find team metadata cache entries whose token no team has, and delete them with --delete"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument(
+ "--delete",
+ action="store_true",
+ help="Delete the orphaned entries. Without this flag the command only lists them.",
+ )
+ parser.add_argument(
+ "--limit",
+ type=int,
+ help="Stop after this many orphaned entries",
+ )
+ parser.add_argument(
+ "--batch-size",
+ type=int,
+ default=1000,
+ help="Number of tokens to check against Postgres per query (default: 1000)",
+ )
+
+ def get_hypercache_config(self) -> HyperCacheManagementConfig:
+ return TEAM_HYPERCACHE_MANAGEMENT_CONFIG
+
+ def handle(self, *args, **options) -> None:
+ if not self.check_dedicated_cache_configured():
+ return
+
+ delete: bool = options["delete"]
+ limit: int | None = options["limit"]
+ batch_size: int = options["batch_size"]
+
+ if limit is not None and limit < 1:
+ raise CommandError("--limit must be at least 1")
+ if not self.validate_batch_size(batch_size):
+ return
+
+ self.print_header("Sweep orphaned team metadata cache entries" + ("" if delete else " (dry run)"))
+ result = sweep_orphaned_token_entries(
+ self.get_hypercache_config().hypercache,
+ delete=delete,
+ limit=limit,
+ batch_size=batch_size,
+ )
+
+ self.stdout.write(f"Tokens checked: {result.tokens_checked}")
+ self.stdout.write(f"Orphaned entries: {len(result.orphaned_tokens)}")
+ for token in result.orphaned_tokens:
+ self.stdout.write(f" {mask_key_value(token)}")
+
+ if not delete:
+ if result.orphaned_tokens:
+ self.stdout.write(self.style.WARNING("Dry run: nothing was deleted. Re-run with --delete to delete."))
+ return
+
+ self.stdout.write(self.style.SUCCESS(f"Deleted: {len(result.orphaned_tokens) - len(result.failed_tokens)}"))
+ if result.failed_tokens:
+ masked = ", ".join(mask_key_value(token) for token in result.failed_tokens)
+ raise CommandError(f"Failed to delete {len(result.failed_tokens)} entries: {masked}")
diff --git a/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.py b/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.py
new file mode 100644
index 000000000000..c79f90dc6e61
--- /dev/null
+++ b/posthog/management/commands/test/test_sweep_orphaned_team_metadata_cache.py
@@ -0,0 +1,135 @@
+import re
+import dataclasses
+from io import StringIO
+from uuid import uuid4
+
+import pytest
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.db import OperationalError
+from django.test import override_settings
+
+from parameterized import parameterized
+
+from posthog.caching.flags_redis_cache import FLAGS_DEDICATED_CACHE_ALIAS
+from posthog.models.utils import mask_key_value
+from posthog.storage import object_storage
+from posthog.storage.hypercache import HyperCache, HyperCacheStoreMissing
+from posthog.storage.team_metadata_cache import TEAM_HYPERCACHE_MANAGEMENT_CONFIG
+
+COMMAND_MODULE = "posthog.management.commands.sweep_orphaned_team_metadata_cache"
+
+
+@override_settings(
+ OBJECT_STORAGE_ENABLED=True,
+ CACHES={
+ "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
+ FLAGS_DEDICATED_CACHE_ALIAS: {
+ "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+ "LOCATION": "redis://stub:6379/",
+ },
+ },
+)
+class TestSweepOrphanedTeamMetadataCache(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ # A namespace of its own keeps the sweep away from entries other tests write to the shared bucket.
+ self.hypercache = HyperCache(
+ namespace=f"sweep_test_{uuid4().hex}",
+ value="full_metadata.json",
+ token_based=True,
+ load_fn=lambda key: HyperCacheStoreMissing(),
+ )
+ config_patch = patch(
+ f"{COMMAND_MODULE}.TEAM_HYPERCACHE_MANAGEMENT_CONFIG",
+ dataclasses.replace(TEAM_HYPERCACHE_MANAGEMENT_CONFIG, hypercache=self.hypercache),
+ )
+ config_patch.start()
+ self.addCleanup(config_patch.stop)
+
+ self.hypercache.set_cache_value(self.team, {"id": self.team.id})
+ self.addCleanup(self.hypercache.delete_cache_entry, self.team.api_token)
+
+ def _write_orphan(self, prefix: str = "phc_orphan_") -> str:
+ token = f"{prefix}{uuid4().hex}"
+ self.hypercache.set_cache_value(token, {"id": 999999})
+ self.addCleanup(self.hypercache.delete_cache_entry, token)
+ return token
+
+ def _entry_exists(self, token: str) -> bool:
+ key = self.hypercache.get_cache_key(token)
+ in_s3 = object_storage.read(key, missing_ok=True) is not None
+ in_redis = self.hypercache.cache_client.get(key) is not None
+ assert in_s3 == in_redis
+ return in_s3
+
+ def _run(self, *args: str) -> str:
+ out = StringIO()
+ call_command("sweep_orphaned_team_metadata_cache", *args, stdout=out)
+ return out.getvalue()
+
+ @parameterized.expand([("dry_run", [], True), ("delete", ["--delete"], False)])
+ def test_sweep_reports_orphans_and_deletes_only_with_flag(
+ self, _name: str, args: list[str], orphan_survives: bool
+ ) -> None:
+ orphans = [self._write_orphan(), self._write_orphan(prefix="phc_custom/")]
+ sibling_cache = HyperCache(
+ namespace=self.hypercache.namespace,
+ value="other.json",
+ token_based=True,
+ load_fn=lambda key: HyperCacheStoreMissing(),
+ )
+ sibling_only_token = f"phc_sibling_{uuid4().hex}"
+ sibling_cache.set_cache_value(sibling_only_token, {"id": 999999})
+ self.addCleanup(sibling_cache.delete_cache_entry, sibling_only_token)
+
+ output = self._run(*args)
+
+ for orphan in orphans:
+ assert mask_key_value(orphan) in output
+ assert orphan not in output
+ assert self._entry_exists(orphan) is orphan_survives
+ assert self.team.api_token not in output
+ assert mask_key_value(sibling_only_token) not in output
+ assert self._entry_exists(self.team.api_token)
+ assert object_storage.read(sibling_cache.get_cache_key(sibling_only_token), missing_ok=True) is not None
+
+ def test_limit_stops_after_n_orphans_across_batches(self) -> None:
+ orphans = [self._write_orphan() for _ in range(3)]
+
+ self._run("--delete", "--limit", "2", "--batch-size", "2")
+
+ assert [self._entry_exists(token) for token in orphans].count(False) == 2
+ assert self._entry_exists(self.team.api_token)
+
+ def test_team_lookup_error_in_a_later_batch_deletes_nothing(self) -> None:
+ orphans = [self._write_orphan(), self._write_orphan()]
+
+ with (
+ patch(
+ "posthog.storage.hypercache_manager._existing_api_tokens",
+ side_effect=[set(), OperationalError("down")],
+ ),
+ pytest.raises(OperationalError),
+ ):
+ self._run("--delete", "--batch-size", "1")
+
+ assert all(self._entry_exists(token) for token in orphans)
+
+ def test_redis_delete_failure_keeps_the_object_for_the_next_sweep(self) -> None:
+ orphan = self._write_orphan()
+
+ with (
+ patch.object(self.hypercache.cache_client, "delete", side_effect=ConnectionError("down")),
+ pytest.raises(CommandError, match=re.escape(mask_key_value(orphan))),
+ ):
+ self._run("--delete")
+
+ assert self._entry_exists(orphan)
+
+ self._run("--delete")
+
+ assert not self._entry_exists(orphan)
diff --git a/posthog/storage/hypercache.py b/posthog/storage/hypercache.py
index d79be7e2772f..5940428d8987 100644
--- a/posthog/storage/hypercache.py
+++ b/posthog/storage/hypercache.py
@@ -1,7 +1,7 @@
import json
import time
import hashlib
-from collections.abc import Callable
+from collections.abc import Callable, Iterator
from typing import Optional
from django.conf import settings
@@ -33,6 +33,8 @@ _REDIS_READ_ERRORS = (
)
+TOKEN_CACHE_KEY_PREFIX = "cache/team_tokens/"
+
DEFAULT_CACHE_MISS_TTL = 60 * 60 * 24 # 1 day - it will be invalidated by the daily sync
DEFAULT_CACHE_TTL = 60 * 60 * 24 * 30 # 30 days
@@ -249,12 +251,29 @@ class HyperCache:
if self.token_based:
if isinstance(key, Team):
key = key.api_token
- return f"cache/team_tokens/{key}/{self.namespace}/{self.value}"
+ return f"{TOKEN_CACHE_KEY_PREFIX}{key}/{self.namespace}/{self.value}"
else:
if isinstance(key, Team):
key = key.id
return f"cache/teams/{key}/{self.namespace}/{self.value}"
+ def iter_object_storage_tokens(self) -> Iterator[str]:
+ """Yield the token of each object-storage entry of this token-based cache.
+
+ All token-based caches share one key prefix. The listing covers that prefix and
+ skips the keys of the other caches. Raises ObjectStorageError when the listing fails.
+ """
+ if not self.token_based:
+ raise ValueError("iter_object_storage_tokens requires a token-based cache")
+ # A custom token can contain "/", so the token is the text between the prefix and this cache's suffix.
+ suffix = f"/{self.namespace}/{self.value}"
+ for key in object_storage.iter_keys(TOKEN_CACHE_KEY_PREFIX):
+ if not key.endswith(suffix):
+ continue
+ token = key.removeprefix(TOKEN_CACHE_KEY_PREFIX).removesuffix(suffix)
+ if token and self.get_cache_key(token) == key:
+ yield token
+
def get_etag_key(self, key: KeyType) -> str:
return f"{self.get_cache_key(key)}:etag"
diff --git a/posthog/storage/hypercache_manager.py b/posthog/storage/hypercache_manager.py
index 2c9cf9b9e3bc..5f4bdf7b29fa 100644
--- a/posthog/storage/hypercache_manager.py
+++ b/posthog/storage/hypercache_manager.py
@@ -13,13 +13,14 @@ Operations include:
import random
import statistics
from collections.abc import Callable
+from itertools import batched
from typing import TYPE_CHECKING, Any, Protocol
if TYPE_CHECKING:
from django.db.models import QuerySet
from django.conf import settings
-from django.db import connection
+from django.db import DEFAULT_DB_ALIAS, connection
import structlog
from posthoganalytics import capture_exception
@@ -28,6 +29,7 @@ from prometheus_client import Counter, Gauge
from posthog.dataclasses import frozen
from posthog.metrics import pushed_metrics_registry
from posthog.models.team.team import Team
+from posthog.models.utils import mask_key_value
from posthog.redis import get_client
from posthog.storage.hypercache import HyperCache
@@ -732,3 +734,64 @@ def batch_check_expiry_tracking(
results[identifier] = score is not None
return results
+
+
+@frozen
+class OrphanedTokenSweepResult:
+ tokens_checked: int
+ orphaned_tokens: tuple[str, ...]
+ failed_tokens: tuple[str, ...]
+
+
+def _existing_api_tokens(tokens: tuple[str, ...]) -> set[str]:
+ # The primary avoids replica lag, which would report a team created moments ago as missing.
+ return set(Team.objects.using(DEFAULT_DB_ALIAS).filter(api_token__in=tokens).values_list("api_token", flat=True))
+
+
+def sweep_orphaned_token_entries(
+ hypercache: HyperCache,
+ *,
+ delete: bool,
+ limit: int | None = None,
+ batch_size: int = 1000,
+) -> OrphanedTokenSweepResult:
+ """Find the object-storage entries of a token-based cache whose token no team has, and optionally delete them.
+
+ An entry is orphaned only when Postgres has no Team row with its token. The payload is never read.
+ The sweep checks every listed token before it deletes any entry.
+ A failed listing or Team lookup raises, so the sweep deletes nothing.
+ A failed delete is logged and recorded in failed_tokens, and the sweep continues.
+ The sweep stops after `limit` orphaned entries.
+ """
+ tokens_checked = 0
+ orphaned: list[str] = []
+ for batch in batched(hypercache.iter_object_storage_tokens(), batch_size, strict=False):
+ tokens_checked += len(batch)
+ existing = _existing_api_tokens(batch)
+ orphaned.extend(token for token in batch if token not in existing)
+ if limit is not None and len(orphaned) >= limit:
+ del orphaned[limit:]
+ break
+
+ failed: list[str] = []
+ if delete:
+ for token in orphaned:
+ masked_token = mask_key_value(token)
+ try:
+ # The sweep finds entries only by listing object storage. The object goes last, so a
+ # failed Redis delete leaves it in place and the next sweep finds the token again.
+ hypercache.delete_cache_entry(token, kinds=["redis"], strict_mirror=True)
+ hypercache.delete_cache_entry(token, kinds=["s3"])
+ logger.info("Deleted orphaned cache entry", namespace=hypercache.namespace, token=masked_token)
+ except Exception as e:
+ logger.exception(
+ "Failed to delete orphaned cache entry", namespace=hypercache.namespace, token=masked_token
+ )
+ capture_exception(e)
+ failed.append(token)
+
+ return OrphanedTokenSweepResult(
+ tokens_checked=tokens_checked,
+ orphaned_tokens=tuple(orphaned),
+ failed_tokens=tuple(failed),
+ )
diff --git a/posthog/storage/object_storage.py b/posthog/storage/object_storage.py
index af38295b7c11..e8a919b5ab5c 100644
--- a/posthog/storage/object_storage.py
+++ b/posthog/storage/object_storage.py
@@ -1,5 +1,6 @@
import abc
import threading
+from collections.abc import Iterator
from typing import IO, Any, Optional, Union
from urllib.parse import urlparse
@@ -59,6 +60,11 @@ class ObjectStorageClient(metaclass=abc.ABCMeta):
def list_objects(self, bucket: str, prefix: str) -> Optional[list[str]]:
pass
+ @abc.abstractmethod
+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:
+ """Yield every key under the prefix, one listing page at a time. Raises ObjectStorageError on any failure."""
+ pass
+
@abc.abstractmethod
def read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:
pass
@@ -140,6 +146,10 @@ class UnavailableStorage(ObjectStorageClient):
def list_objects(self, bucket: str, prefix: str) -> Optional[list[str]]:
pass
+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:
+ # This raises instead of yielding nothing, because an empty listing tells the caller that the prefix holds no keys.
+ raise ObjectStorageError("object storage is unavailable")
+
def read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:
return None
@@ -287,6 +297,19 @@ class ObjectStorage(ObjectStorageClient):
capture_exception(e)
return None
+ def iter_keys(self, bucket: str, prefix: str, *, page_size: int = 1000) -> Iterator[str]:
+ pages = self.aws_client.get_paginator("list_objects_v2").paginate(
+ Bucket=bucket, Prefix=prefix, PaginationConfig={"PageSize": page_size}
+ )
+ try:
+ for page in pages:
+ for obj in page.get("Contents", []):
+ yield obj["Key"]
+ except Exception as e:
+ logger.exception("object_storage.iter_keys_failed", bucket=bucket, prefix=prefix, error=e)
+ capture_exception(e)
+ raise ObjectStorageError("list failed") from e
+
def read(self, bucket: str, key: str, *, missing_ok: bool = False) -> Optional[str]:
object_bytes = self.read_bytes(bucket, key, missing_ok=missing_ok)
if object_bytes:
@@ -597,6 +620,12 @@ def list_objects(prefix: str, bucket: str | None = None) -> Optional[list[str]]:
return object_storage_client().list_objects(bucket=bucket or settings.OBJECT_STORAGE_BUCKET, prefix=prefix)
+def iter_keys(prefix: str, bucket: str | None = None, *, page_size: int = 1000) -> Iterator[str]:
+ return object_storage_client().iter_keys(
+ bucket=bucket or settings.OBJECT_STORAGE_BUCKET, prefix=prefix, page_size=page_size
+ )
+
+
def copy_objects(source_prefix: str, target_prefix: str) -> int:
return (
object_storage_client().copy_objects(
diff --git a/posthog/storage/test/test_object_storage.py b/posthog/storage/test/test_object_storage.py
index 32b500619c9f..ea6fae63b1bb 100644
--- a/posthog/storage/test/test_object_storage.py
+++ b/posthog/storage/test/test_object_storage.py
@@ -30,6 +30,7 @@ from posthog.storage.object_storage import (
get_presigned_url,
health_check,
is_usable_endpoint,
+ iter_keys,
list_objects,
object_storage_client,
read,
@@ -164,6 +165,7 @@ class TestStorage(APIBaseTest):
"test_storage_bucket/a_shared_prefix/b",
"test_storage_bucket/a_shared_prefix/c",
]
+ assert list(iter_keys(prefix=f"{TEST_BUCKET}/{shared_prefix}", page_size=2)) == listing
@patch("posthog.storage.object_storage.capture_exception")
def test_can_list_unknown_prefix(self, mock_capture_exception) -> None: