:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): cache the realtime cohort access level in team metadata (#129562)
Matheus Batistabd008cdbc415
diff --git a/docs/internal/realtime-cohorts/membership-output-and-readers.md b/docs/internal/realtime-cohorts/membership-output-and-readers.mdindex 0cc6c7996a40..5b949d364b0e 100644--- a/docs/internal/realtime-cohorts/membership-output-and-readers.md+++ b/docs/internal/realtime-cohorts/membership-output-and-readers.md@@ -216,6 +216,11 @@ A cohort whose `condition_type` is empty is never routed, and resaving it fillsEvery other non-static cohort is evaluated dynamically from the person's properties, where a behavioral condition can never match.+The team metadata cache that the flags service reads carries the team's access row level as `realtime_cohort_level`, or `off` when the team has no row.+It carries the row's level, not Django's effective level, because `REALTIME_COHORT_TEAM_ALLOWLIST` can name every team.+`realtime_cohort_team_access grant` and `revoke` queue a refresh of the team's entry after they commit.+The hourly team metadata verification repairs an entry whose level still differs from the row.+### How it reads1. The service collects every routed cohort among the cohorts in the team's flags payload, or among all the team's cohorts when the payload carries none.diff --git a/posthog/storage/team_metadata_cache.py b/posthog/storage/team_metadata_cache.pyindex 3f74e6f2ebb2..fcaf3a9f3a57 100644--- a/posthog/storage/team_metadata_cache.py+++ b/posthog/storage/team_metadata_cache.py@@ -65,6 +65,7 @@ from posthog.storage.hypercache_manager import (get_cache_stats as get_cache_stats_generic,)+from products.cohorts.backend.models.team_access import RealtimeCohortLevel, RealtimeCohortTeamAccessfrom products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfigfrom products.feature_flags.backend.models.team_feature_flags_config import (PropertyMatchingVersion,@@ -160,7 +161,11 @@ def _serialize_team_field(field: str, value: Any) -> Any:return value-REQUEST_BLOCKLIST_FIELDS = ("flags_blocked_hosts", "flags_blocked_ips")+MISSING_KEY_DEFAULTS: dict[str, Any] = {+ "flags_blocked_hosts": [],+ "flags_blocked_ips": [],+ "realtime_cohort_level": RealtimeCohortLevel.OFF.value,+}def _load_request_blocklists(team_ids: list[int]) -> dict[int, dict[str, list[str]]]:@@ -175,11 +180,25 @@ def _load_request_blocklists(team_ids: list[int]) -> dict[int, dict[str, list[streturn blocklists+def _load_realtime_cohort_levels(team_ids: list[int]) -> dict[int, str]:+ """Map each team id to the level of its realtime cohort access row, with `off` when it has no row.++ The flags service routes a team's realtime cohorts to the membership table at `serving`. It reads+ the row's level from Postgres too, so the cache carries the row's level and the two reads agree.+ Django's effective level also counts REALTIME_COHORT_TEAM_ALLOWLIST, which can name every team.+ """+ levels = dict.fromkeys(team_ids, RealtimeCohortLevel.OFF.value)+ for team_id, level in RealtimeCohortTeamAccess.objects.filter(team_id__in=team_ids).values_list("team_id", "level"):+ levels[team_id] = level+ return levels++def _serialize_team_to_metadata(team: Team,minimal_flag_called_events: bool | None = None,property_matching_version: int | None = None,request_blocklist: dict[str, list[str]] | None = None,+ realtime_cohort_level: str | None = None,) -> dict[str, Any]:"""Serialize a Team object to metadata dictionary.@@ -192,6 +211,8 @@ def _serialize_team_to_metadata(this looks it up with the other feature flags config value.request_blocklist: Pre-fetched entry from _load_request_blocklists. If None, thislooks it up itself.+ realtime_cohort_level: Pre-fetched entry from _load_realtime_cohort_levels. If None,+ this looks it up itself.Returns:Dictionary containing full team metadata@@ -225,6 +246,10 @@ def _serialize_team_to_metadata(request_blocklist = _load_request_blocklists([team.id])[team.id]metadata.update(request_blocklist)+ if realtime_cohort_level is None:+ realtime_cohort_level = _load_realtime_cohort_levels([team.id])[team.id]+ metadata["realtime_cohort_level"] = realtime_cohort_level+return metadata@@ -249,12 +274,14 @@ def _batch_load_team_metadata(teams: list[Team]) -> dict[int, dict[str, Any]]:).values_list("team_id", "minimal_flag_called_events", "property_matching_version")}blocklist_by_team_id = _load_request_blocklists([team.id for team in teams])+ level_by_team_id = _load_realtime_cohort_levels([team.id for team in teams])return {team.id: _serialize_team_to_metadata(team,minimal_flag_called_events=config_by_team_id.get(team.id, (False, PropertyMatchingVersion.LEGACY))[0],property_matching_version=config_by_team_id.get(team.id, (False, PropertyMatchingVersion.LEGACY))[1],request_blocklist=blocklist_by_team_id[team.id],+ realtime_cohort_level=level_by_team_id[team.id],)for team in teams}@@ -377,14 +404,15 @@ def verify_team_metadata("project_name","minimal_flag_called_events","property_matching_version",- *REQUEST_BLOCKLIST_FIELDS,+ *MISSING_KEY_DEFAULTS,}diffs = []for key in fields_to_check:db_val = db_data.get(key)- # A blob without the blocklist keys reads the same as empty lists in the flags service,- # so it only needs a fix when the team actually has entries.- cached_val = cached_data.get(key, [] if key in REQUEST_BLOCKLIST_FIELDS else None)+ # A blob without one of these keys reads as this value in the flags service, so it only+ # needs a fix when the team's value differs. Otherwise the first verification after a new+ # key ships would rewrite every team's entry.+ cached_val = cached_data.get(key, MISSING_KEY_DEFAULTS.get(key))if db_val != cached_val:diffs.append({"field": key, "db_value": db_val, "cached_value": cached_val})diff --git a/posthog/storage/team_metadata_signal_handlers.py b/posthog/storage/team_metadata_signal_handlers.pyindex 5bb67e9ac588..75a0991ea6b4 100644--- a/posthog/storage/team_metadata_signal_handlers.py+++ b/posthog/storage/team_metadata_signal_handlers.py@@ -24,6 +24,7 @@ from posthog.models.team import Teamfrom posthog.storage.hypercache_manager import HYPERCACHE_SIGNAL_UPDATE_COUNTERfrom posthog.storage.team_metadata_cache import clear_team_metadata_cache+from products.cohorts.backend.models.team_access import RealtimeCohortTeamAccessfrom products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfiglogger = structlog.get_logger(__name__)@@ -104,6 +105,18 @@ def update_team_metadata_cache_on_flag_policy_save(_enqueue_team_metadata_update_on_commit(instance.team_id)+def update_team_metadata_cache_on_realtime_cohort_access_save(+ sender: type[RealtimeCohortTeamAccess], instance: RealtimeCohortTeamAccess, created: bool, **kwargs: Any+) -> None:+ """Refresh the cached team when its realtime cohort level changes, because the flags service reads it from that cache."""+ if not settings.FLAGS_REDIS_URL:+ return+ if not _any_field_may_have_changed(kwargs.get("update_fields"), frozenset({"level"})):+ return++ _enqueue_team_metadata_update_on_commit(instance.team_id)++def _enqueue_deleted_team_metadata_clear_on_commit(team_id: int, api_token: str) -> None:# The pre_delete clear runs before the commit and does not retry. A refresh in that gap can write the# entry back. The S3 copy has no TTL, so a retrying task clears the entry again after the commit.@@ -161,6 +174,7 @@ def update_team_metadata_cache_on_project_save(def connect_signal_handlers() -> None:post_save.connect(update_team_metadata_cache_on_save, sender=Team)post_save.connect(update_team_metadata_cache_on_flag_policy_save, sender=TeamFeatureFlagPolicyConfig)+ post_save.connect(update_team_metadata_cache_on_realtime_cohort_access_save, sender=RealtimeCohortTeamAccess)pre_delete.connect(clear_team_metadata_cache_on_delete, sender=Team)post_save.connect(update_team_metadata_cache_on_organization_save, sender=Organization)post_save.connect(update_team_metadata_cache_on_project_save, sender=Project)diff --git a/posthog/storage/test/test_team_metadata_cache.py b/posthog/storage/test/test_team_metadata_cache.pyindex 1e1eaf094850..6dab879b0756 100644--- a/posthog/storage/test/test_team_metadata_cache.py+++ b/posthog/storage/test/test_team_metadata_cache.py@@ -27,6 +27,11 @@ from posthog.storage.team_metadata_cache import ()from posthog.tasks.team_metadata import update_team_metadata_cache_task+from products.cohorts.backend.models.team_access import (+ RealtimeCohortAccessPurpose,+ RealtimeCohortLevel,+ RealtimeCohortTeamAccess,+)from products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfigfrom products.feature_flags.backend.models.team_feature_flags_config import TeamFeatureFlagsConfig@@ -451,6 +456,7 @@ class TestVerifyTeamMetadata(BaseTest):("property_matching_version", 2),("flags_blocked_hosts", ["example.com"]),("flags_blocked_ips", ["203.0.113.7"]),+ ("realtime_cohort_level", "serving"),])@patch("posthog.storage.team_metadata_cache.get_team_metadata")@@ -469,22 +475,33 @@ class TestVerifyTeamMetadata(BaseTest):self.assertEqual(result["issue"], "DATA_MISMATCH")self.assertIn(field, result["diff_fields"])- @parameterized.expand([("without_entries", [], "match"), ("with_entries", ["example.com"], "mismatch")])+ @parameterized.expand(+ [+ ("team_without_values", False, []),+ ("team_with_values", True, ["flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"]),+ ]+ )@patch("posthog.storage.team_metadata_cache.get_team_metadata")- def test_verify_reads_missing_blocklist_keys_as_empty(- self, _name, blocked_hosts, expected_status, mock_get_metadata- ):+ def test_verify_reads_missing_keys_as_their_defaults(+ self, _name: str, team_has_values: bool, expected_diff_fields: list[str], mock_get_metadata: MagicMock+ ) -> None:from posthog.storage.team_metadata_cache import _serialize_team_to_metadata- TeamFeatureFlagPolicyConfig.objects.create(team=self.team, blocked_hosts=blocked_hosts)+ if team_has_values:+ TeamFeatureFlagPolicyConfig.objects.create(+ team=self.team, blocked_hosts=["example.com"], blocked_ips=["203.0.113.7"]+ )+ RealtimeCohortTeamAccess.objects.create(+ team=self.team, level=RealtimeCohortLevel.SERVING, purpose=RealtimeCohortAccessPurpose.PILOT+ )cached_data = _serialize_team_to_metadata(self.team)- del cached_data["flags_blocked_hosts"]- del cached_data["flags_blocked_ips"]+ for key in ("flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"):+ del cached_data[key]mock_get_metadata.return_value = cached_dataresult = verify_team_metadata(self.team)- self.assertEqual(result["status"], expected_status)+ self.assertEqual(result.get("diff_fields", []), expected_diff_fields)@patch("posthog.storage.team_metadata_cache.get_team_metadata")def test_verify_returns_miss_when_no_cached_data(self, mock_get_metadata):@@ -534,12 +551,15 @@ class TestFeatureFlagsConfigInMetadata(BaseTest):TeamFeatureFlagPolicyConfig.objects.create(team=gated_team, blocked_hosts=["example.com"], blocked_ips=["203.0.113.0/24"])+ RealtimeCohortTeamAccess.objects.create(+ team=gated_team, level=RealtimeCohortLevel.SERVING, purpose=RealtimeCohortAccessPurpose.PILOT+ )ungated_team = self.organization.teams.create(name="Ungated team")teams = list(Team.objects.select_related("organization", "project").filter(id__in=[gated_team.id, ungated_team.id]))- with self.assertNumQueries(2):+ with self.assertNumQueries(3):batch_result = _batch_load_team_metadata(teams)self.assertEqual(@@ -556,11 +576,13 @@ class TestFeatureFlagsConfigInMetadata(BaseTest):self.assertEqual(batch_result[ungated_team.id]["property_matching_version"], 1)for team in (gated_team, ungated_team):single_result = _serialize_team_to_metadata(team)- for field in ("flags_blocked_hosts", "flags_blocked_ips"):+ for field in ("flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"):self.assertEqual(batch_result[team.id][field], single_result[field])self.assertEqual(batch_result[gated_team.id]["flags_blocked_hosts"], ["example.com"])self.assertEqual(batch_result[gated_team.id]["flags_blocked_ips"], ["203.0.113.0/24"])self.assertEqual(batch_result[ungated_team.id]["flags_blocked_hosts"], [])+ self.assertEqual(batch_result[gated_team.id]["realtime_cohort_level"], "serving")+ self.assertEqual(batch_result[ungated_team.id]["realtime_cohort_level"], "off")@override_settings(FLAGS_REDIS_URL="redis://test:6379/0")diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txtindex 43467b5cf170..c6d381894c13 100644--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt@@ -187,6 +187,7 @@ post_save:posthog.storage.team_llm_gateway_policy_signal_handlers._update_cache_post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_flag_policy_savepost_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_organization_savepost_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_project_save+post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_realtime_cohort_access_savepost_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_savepost_save:products.access_control.backend.property_access_control._invalidate_restriction_cache_on_changepost_save:products.actions.backend.models.action.action_saveddiff --git a/products/cohorts/backend/realtime_teams.py b/products/cohorts/backend/realtime_teams.pyindex 1bbe5c740dcd..a1df4f94c298 100644--- a/products/cohorts/backend/realtime_teams.py+++ b/products/cohorts/backend/realtime_teams.py@@ -228,16 +228,18 @@ def revoke_realtime_cohort_access(rows: QuerySet[RealtimeCohortTeamAccess]) -> lthe row says, so an `off` row would tell every reader of the table the opposite of the truth."""with transaction.atomic():- revoked = sorted(- rows.exclude(level=RealtimeCohortLevel.OFF).select_for_update().values_list("team_id", flat=True)- )+ revoked_rows = list(rows.exclude(level=RealtimeCohortLevel.OFF).select_for_update().order_by("team_id"))+ revoked = [row.team_id for row in revoked_rows]_refuse("On REALTIME_COHORT_TEAM_ALLOWLIST", [team_id for team_id in revoked if realtime_allowlist_matches(team_id)])now = timezone.now()- RealtimeCohortTeamAccess.objects.filter(team_id__in=revoked).update(- level=RealtimeCohortLevel.OFF, off_since=now, changed_at=now- )+ for row in revoked_rows:+ row.level = RealtimeCohortLevel.OFF+ row.off_since = now+ # A save per row, not QuerySet.update(), because post_save refreshes the team metadata cache,+ # and the flags service reads the level from that cache.+ row.save(update_fields=["level", "off_since", "changed_at"])return revokeddiff --git a/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py b/products/cohorts/backend/test/test_realtime_cohort_team_access_command.pyindex 9d90ca89021e..064856052c16 100644--- a/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py+++ b/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py@@ -3,6 +3,7 @@ from io import StringIOimport time_machinefrom posthog.test.base import BaseTest+from unittest.mock import patchfrom django.core.management import call_commandfrom django.core.management.base import CommandError@@ -78,6 +79,21 @@ class TestRealtimeCohortTeamAccessCommand(BaseTest):call_command("realtime_cohort_team_access", "list", stdout=out)assert f"{self.team.pk}\toff\tload_test\tTrue" in out.getvalue()+ @parameterized.expand([("first_grant", None), ("promotion", "processing"), ("revoke", "serving")])+ @override_settings(FLAGS_REDIS_URL="redis://localhost:6379")+ def test_level_changes_refresh_the_team_metadata_cache(self, change: str, existing_level: str | None) -> None:+ if existing_level is not None:+ self._grant(self.team.pk, level=existing_level)++ with patch("posthog.tasks.team_metadata.update_team_metadata_cache_task") as mock_task:+ with self.captureOnCommitCallbacks(execute=True):+ if change == "revoke":+ call_command("realtime_cohort_team_access", "revoke", "--teams", str(self.team.pk))+ else:+ self._grant(self.team.pk, level="serving")++ mock_task.delay.assert_called_once_with(self.team.pk)+@parameterized.expand([("unknown_team", "No team with id"),