Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): cache the realtime cohort access level in team metadata (#129562)

Matheus Batistabd008cdbc415
diff --git a/docs/internal/realtime-cohorts/membership-output-and-readers.md b/docs/internal/realtime-cohorts/membership-output-and-readers.md
index 0cc6c7996a40..5b949d364b0e 100644
--- a/docs/internal/realtime-cohorts/membership-output-and-readers.md
+++ b/docs/internal/realtime-cohorts/membership-output-and-readers.md
@@ -216,6 +216,11 @@ A cohort whose `condition_type` is empty is never routed, and resaving it fills
Every other non-static cohort is evaluated dynamically from the person's properties, where a behavioral condition can never match.
+The team metadata cache that the flags service reads carries the team's access row level as `realtime_cohort_level`, or `off` when the team has no row.
+It carries the row's level, not Django's effective level, because `REALTIME_COHORT_TEAM_ALLOWLIST` can name every team.
+`realtime_cohort_team_access grant` and `revoke` queue a refresh of the team's entry after they commit.
+The hourly team metadata verification repairs an entry whose level still differs from the row.
+
### How it reads
1. The service collects every routed cohort among the cohorts in the team's flags payload, or among all the team's cohorts when the payload carries none.
diff --git a/posthog/storage/team_metadata_cache.py b/posthog/storage/team_metadata_cache.py
index 3f74e6f2ebb2..fcaf3a9f3a57 100644
--- a/posthog/storage/team_metadata_cache.py
+++ b/posthog/storage/team_metadata_cache.py
@@ -65,6 +65,7 @@ from posthog.storage.hypercache_manager import (
get_cache_stats as get_cache_stats_generic,
)
+from products.cohorts.backend.models.team_access import RealtimeCohortLevel, RealtimeCohortTeamAccess
from products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfig
from products.feature_flags.backend.models.team_feature_flags_config import (
PropertyMatchingVersion,
@@ -160,7 +161,11 @@ def _serialize_team_field(field: str, value: Any) -> Any:
return value
-REQUEST_BLOCKLIST_FIELDS = ("flags_blocked_hosts", "flags_blocked_ips")
+MISSING_KEY_DEFAULTS: dict[str, Any] = {
+ "flags_blocked_hosts": [],
+ "flags_blocked_ips": [],
+ "realtime_cohort_level": RealtimeCohortLevel.OFF.value,
+}
def _load_request_blocklists(team_ids: list[int]) -> dict[int, dict[str, list[str]]]:
@@ -175,11 +180,25 @@ def _load_request_blocklists(team_ids: list[int]) -> dict[int, dict[str, list[st
return blocklists
+def _load_realtime_cohort_levels(team_ids: list[int]) -> dict[int, str]:
+ """Map each team id to the level of its realtime cohort access row, with `off` when it has no row.
+
+ The flags service routes a team's realtime cohorts to the membership table at `serving`. It reads
+ the row's level from Postgres too, so the cache carries the row's level and the two reads agree.
+ Django's effective level also counts REALTIME_COHORT_TEAM_ALLOWLIST, which can name every team.
+ """
+ levels = dict.fromkeys(team_ids, RealtimeCohortLevel.OFF.value)
+ for team_id, level in RealtimeCohortTeamAccess.objects.filter(team_id__in=team_ids).values_list("team_id", "level"):
+ levels[team_id] = level
+ return levels
+
+
def _serialize_team_to_metadata(
team: Team,
minimal_flag_called_events: bool | None = None,
property_matching_version: int | None = None,
request_blocklist: dict[str, list[str]] | None = None,
+ realtime_cohort_level: str | None = None,
) -> dict[str, Any]:
"""
Serialize a Team object to metadata dictionary.
@@ -192,6 +211,8 @@ def _serialize_team_to_metadata(
this looks it up with the other feature flags config value.
request_blocklist: Pre-fetched entry from _load_request_blocklists. If None, this
looks it up itself.
+ realtime_cohort_level: Pre-fetched entry from _load_realtime_cohort_levels. If None,
+ this looks it up itself.
Returns:
Dictionary containing full team metadata
@@ -225,6 +246,10 @@ def _serialize_team_to_metadata(
request_blocklist = _load_request_blocklists([team.id])[team.id]
metadata.update(request_blocklist)
+ if realtime_cohort_level is None:
+ realtime_cohort_level = _load_realtime_cohort_levels([team.id])[team.id]
+ metadata["realtime_cohort_level"] = realtime_cohort_level
+
return metadata
@@ -249,12 +274,14 @@ def _batch_load_team_metadata(teams: list[Team]) -> dict[int, dict[str, Any]]:
).values_list("team_id", "minimal_flag_called_events", "property_matching_version")
}
blocklist_by_team_id = _load_request_blocklists([team.id for team in teams])
+ level_by_team_id = _load_realtime_cohort_levels([team.id for team in teams])
return {
team.id: _serialize_team_to_metadata(
team,
minimal_flag_called_events=config_by_team_id.get(team.id, (False, PropertyMatchingVersion.LEGACY))[0],
property_matching_version=config_by_team_id.get(team.id, (False, PropertyMatchingVersion.LEGACY))[1],
request_blocklist=blocklist_by_team_id[team.id],
+ realtime_cohort_level=level_by_team_id[team.id],
)
for team in teams
}
@@ -377,14 +404,15 @@ def verify_team_metadata(
"project_name",
"minimal_flag_called_events",
"property_matching_version",
- *REQUEST_BLOCKLIST_FIELDS,
+ *MISSING_KEY_DEFAULTS,
}
diffs = []
for key in fields_to_check:
db_val = db_data.get(key)
- # A blob without the blocklist keys reads the same as empty lists in the flags service,
- # so it only needs a fix when the team actually has entries.
- cached_val = cached_data.get(key, [] if key in REQUEST_BLOCKLIST_FIELDS else None)
+ # A blob without one of these keys reads as this value in the flags service, so it only
+ # needs a fix when the team's value differs. Otherwise the first verification after a new
+ # key ships would rewrite every team's entry.
+ cached_val = cached_data.get(key, MISSING_KEY_DEFAULTS.get(key))
if db_val != cached_val:
diffs.append({"field": key, "db_value": db_val, "cached_value": cached_val})
diff --git a/posthog/storage/team_metadata_signal_handlers.py b/posthog/storage/team_metadata_signal_handlers.py
index 5bb67e9ac588..75a0991ea6b4 100644
--- a/posthog/storage/team_metadata_signal_handlers.py
+++ b/posthog/storage/team_metadata_signal_handlers.py
@@ -24,6 +24,7 @@ from posthog.models.team import Team
from posthog.storage.hypercache_manager import HYPERCACHE_SIGNAL_UPDATE_COUNTER
from posthog.storage.team_metadata_cache import clear_team_metadata_cache
+from products.cohorts.backend.models.team_access import RealtimeCohortTeamAccess
from products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfig
logger = structlog.get_logger(__name__)
@@ -104,6 +105,18 @@ def update_team_metadata_cache_on_flag_policy_save(
_enqueue_team_metadata_update_on_commit(instance.team_id)
+def update_team_metadata_cache_on_realtime_cohort_access_save(
+ sender: type[RealtimeCohortTeamAccess], instance: RealtimeCohortTeamAccess, created: bool, **kwargs: Any
+) -> None:
+ """Refresh the cached team when its realtime cohort level changes, because the flags service reads it from that cache."""
+ if not settings.FLAGS_REDIS_URL:
+ return
+ if not _any_field_may_have_changed(kwargs.get("update_fields"), frozenset({"level"})):
+ return
+
+ _enqueue_team_metadata_update_on_commit(instance.team_id)
+
+
def _enqueue_deleted_team_metadata_clear_on_commit(team_id: int, api_token: str) -> None:
# The pre_delete clear runs before the commit and does not retry. A refresh in that gap can write the
# entry back. The S3 copy has no TTL, so a retrying task clears the entry again after the commit.
@@ -161,6 +174,7 @@ def update_team_metadata_cache_on_project_save(
def connect_signal_handlers() -> None:
post_save.connect(update_team_metadata_cache_on_save, sender=Team)
post_save.connect(update_team_metadata_cache_on_flag_policy_save, sender=TeamFeatureFlagPolicyConfig)
+ post_save.connect(update_team_metadata_cache_on_realtime_cohort_access_save, sender=RealtimeCohortTeamAccess)
pre_delete.connect(clear_team_metadata_cache_on_delete, sender=Team)
post_save.connect(update_team_metadata_cache_on_organization_save, sender=Organization)
post_save.connect(update_team_metadata_cache_on_project_save, sender=Project)
diff --git a/posthog/storage/test/test_team_metadata_cache.py b/posthog/storage/test/test_team_metadata_cache.py
index 1e1eaf094850..6dab879b0756 100644
--- a/posthog/storage/test/test_team_metadata_cache.py
+++ b/posthog/storage/test/test_team_metadata_cache.py
@@ -27,6 +27,11 @@ from posthog.storage.team_metadata_cache import (
)
from posthog.tasks.team_metadata import update_team_metadata_cache_task
+from products.cohorts.backend.models.team_access import (
+ RealtimeCohortAccessPurpose,
+ RealtimeCohortLevel,
+ RealtimeCohortTeamAccess,
+)
from products.feature_flags.backend.models.team_feature_flag_policy_config import TeamFeatureFlagPolicyConfig
from products.feature_flags.backend.models.team_feature_flags_config import TeamFeatureFlagsConfig
@@ -451,6 +456,7 @@ class TestVerifyTeamMetadata(BaseTest):
("property_matching_version", 2),
("flags_blocked_hosts", ["example.com"]),
("flags_blocked_ips", ["203.0.113.7"]),
+ ("realtime_cohort_level", "serving"),
]
)
@patch("posthog.storage.team_metadata_cache.get_team_metadata")
@@ -469,22 +475,33 @@ class TestVerifyTeamMetadata(BaseTest):
self.assertEqual(result["issue"], "DATA_MISMATCH")
self.assertIn(field, result["diff_fields"])
- @parameterized.expand([("without_entries", [], "match"), ("with_entries", ["example.com"], "mismatch")])
+ @parameterized.expand(
+ [
+ ("team_without_values", False, []),
+ ("team_with_values", True, ["flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"]),
+ ]
+ )
@patch("posthog.storage.team_metadata_cache.get_team_metadata")
- def test_verify_reads_missing_blocklist_keys_as_empty(
- self, _name, blocked_hosts, expected_status, mock_get_metadata
- ):
+ def test_verify_reads_missing_keys_as_their_defaults(
+ self, _name: str, team_has_values: bool, expected_diff_fields: list[str], mock_get_metadata: MagicMock
+ ) -> None:
from posthog.storage.team_metadata_cache import _serialize_team_to_metadata
- TeamFeatureFlagPolicyConfig.objects.create(team=self.team, blocked_hosts=blocked_hosts)
+ if team_has_values:
+ TeamFeatureFlagPolicyConfig.objects.create(
+ team=self.team, blocked_hosts=["example.com"], blocked_ips=["203.0.113.7"]
+ )
+ RealtimeCohortTeamAccess.objects.create(
+ team=self.team, level=RealtimeCohortLevel.SERVING, purpose=RealtimeCohortAccessPurpose.PILOT
+ )
cached_data = _serialize_team_to_metadata(self.team)
- del cached_data["flags_blocked_hosts"]
- del cached_data["flags_blocked_ips"]
+ for key in ("flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"):
+ del cached_data[key]
mock_get_metadata.return_value = cached_data
result = verify_team_metadata(self.team)
- self.assertEqual(result["status"], expected_status)
+ self.assertEqual(result.get("diff_fields", []), expected_diff_fields)
@patch("posthog.storage.team_metadata_cache.get_team_metadata")
def test_verify_returns_miss_when_no_cached_data(self, mock_get_metadata):
@@ -534,12 +551,15 @@ class TestFeatureFlagsConfigInMetadata(BaseTest):
TeamFeatureFlagPolicyConfig.objects.create(
team=gated_team, blocked_hosts=["example.com"], blocked_ips=["203.0.113.0/24"]
)
+ RealtimeCohortTeamAccess.objects.create(
+ team=gated_team, level=RealtimeCohortLevel.SERVING, purpose=RealtimeCohortAccessPurpose.PILOT
+ )
ungated_team = self.organization.teams.create(name="Ungated team")
teams = list(
Team.objects.select_related("organization", "project").filter(id__in=[gated_team.id, ungated_team.id])
)
- with self.assertNumQueries(2):
+ with self.assertNumQueries(3):
batch_result = _batch_load_team_metadata(teams)
self.assertEqual(
@@ -556,11 +576,13 @@ class TestFeatureFlagsConfigInMetadata(BaseTest):
self.assertEqual(batch_result[ungated_team.id]["property_matching_version"], 1)
for team in (gated_team, ungated_team):
single_result = _serialize_team_to_metadata(team)
- for field in ("flags_blocked_hosts", "flags_blocked_ips"):
+ for field in ("flags_blocked_hosts", "flags_blocked_ips", "realtime_cohort_level"):
self.assertEqual(batch_result[team.id][field], single_result[field])
self.assertEqual(batch_result[gated_team.id]["flags_blocked_hosts"], ["example.com"])
self.assertEqual(batch_result[gated_team.id]["flags_blocked_ips"], ["203.0.113.0/24"])
self.assertEqual(batch_result[ungated_team.id]["flags_blocked_hosts"], [])
+ self.assertEqual(batch_result[gated_team.id]["realtime_cohort_level"], "serving")
+ self.assertEqual(batch_result[ungated_team.id]["realtime_cohort_level"], "off")
@override_settings(FLAGS_REDIS_URL="redis://test:6379/0")
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index 43467b5cf170..c6d381894c13 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -187,6 +187,7 @@ post_save:posthog.storage.team_llm_gateway_policy_signal_handlers._update_cache_
post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_flag_policy_save
post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_organization_save
post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_project_save
+post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_realtime_cohort_access_save
post_save:posthog.storage.team_metadata_signal_handlers.update_team_metadata_cache_on_save
post_save:products.access_control.backend.property_access_control._invalidate_restriction_cache_on_change
post_save:products.actions.backend.models.action.action_saved
diff --git a/products/cohorts/backend/realtime_teams.py b/products/cohorts/backend/realtime_teams.py
index 1bbe5c740dcd..a1df4f94c298 100644
--- a/products/cohorts/backend/realtime_teams.py
+++ b/products/cohorts/backend/realtime_teams.py
@@ -228,16 +228,18 @@ def revoke_realtime_cohort_access(rows: QuerySet[RealtimeCohortTeamAccess]) -> l
the row says, so an `off` row would tell every reader of the table the opposite of the truth.
"""
with transaction.atomic():
- revoked = sorted(
- rows.exclude(level=RealtimeCohortLevel.OFF).select_for_update().values_list("team_id", flat=True)
- )
+ revoked_rows = list(rows.exclude(level=RealtimeCohortLevel.OFF).select_for_update().order_by("team_id"))
+ revoked = [row.team_id for row in revoked_rows]
_refuse(
"On REALTIME_COHORT_TEAM_ALLOWLIST", [team_id for team_id in revoked if realtime_allowlist_matches(team_id)]
)
now = timezone.now()
- RealtimeCohortTeamAccess.objects.filter(team_id__in=revoked).update(
- level=RealtimeCohortLevel.OFF, off_since=now, changed_at=now
- )
+ for row in revoked_rows:
+ row.level = RealtimeCohortLevel.OFF
+ row.off_since = now
+ # A save per row, not QuerySet.update(), because post_save refreshes the team metadata cache,
+ # and the flags service reads the level from that cache.
+ row.save(update_fields=["level", "off_since", "changed_at"])
return revoked
diff --git a/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py b/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py
index 9d90ca89021e..064856052c16 100644
--- a/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py
+++ b/products/cohorts/backend/test/test_realtime_cohort_team_access_command.py
@@ -3,6 +3,7 @@ from io import StringIO
import time_machine
from posthog.test.base import BaseTest
+from unittest.mock import patch
from django.core.management import call_command
from django.core.management.base import CommandError
@@ -78,6 +79,21 @@ class TestRealtimeCohortTeamAccessCommand(BaseTest):
call_command("realtime_cohort_team_access", "list", stdout=out)
assert f"{self.team.pk}\toff\tload_test\tTrue" in out.getvalue()
+ @parameterized.expand([("first_grant", None), ("promotion", "processing"), ("revoke", "serving")])
+ @override_settings(FLAGS_REDIS_URL="redis://localhost:6379")
+ def test_level_changes_refresh_the_team_metadata_cache(self, change: str, existing_level: str | None) -> None:
+ if existing_level is not None:
+ self._grant(self.team.pk, level=existing_level)
+
+ with patch("posthog.tasks.team_metadata.update_team_metadata_cache_task") as mock_task:
+ with self.captureOnCommitCallbacks(execute=True):
+ if change == "revoke":
+ call_command("realtime_cohort_team_access", "revoke", "--teams", str(self.team.pk))
+ else:
+ self._grant(self.team.pk, level="serving")
+
+ mock_task.delay.assert_called_once_with(self.team.pk)
+
@parameterized.expand(
[
("unknown_team", "No team with id"),