:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): skip teams without flag requests in the usage export (#122257)
Alex Mayse0e446e19e0b
diff --git a/docs/internal/feature-flags/billing.md b/docs/internal/feature-flags/billing.mdindex 0c1d264aa05c..686318daff4f 100644--- a/docs/internal/feature-flags/billing.md+++ b/docs/internal/feature-flags/billing.md@@ -185,7 +185,13 @@ sender.add_periodic_task(### Aggregation process-For each team (excluding internal metrics and demo teams):+The task reads the teams 1,000 at a time, excluding internal metrics and demo teams.+For each batch, one pipelined `EXISTS` call finds the teams that have a team key for any request type.+The task skips the teams with no team key.+Those teams also have no SDK keys, because the service writes the SDK key in the same pipeline as the team key.+When the `EXISTS` call fails, the task reports the error and exports every team in that batch.++For each team with a team key:1. **Acquire distributed lock**: `posthog:decide_analytics:lock:{team_id}` with 60-second timeout2. **Extract counters**: Read all time buckets except the current one (still being filled)diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.pyindex 9b348956c82c..8c9fee6d5da5 100644--- a/posthog/tasks/tasks.py+++ b/posthog/tasks/tasks.py@@ -1032,8 +1032,10 @@ def calculate_decide_usage() -> None:ph_client = get_regional_ph_client()if ph_client:- capture_decide_usage_for_all_teams(ph_client)- ph_client.shutdown()+ try:+ capture_decide_usage_for_all_teams(ph_client)+ finally:+ ph_client.shutdown()@shared_task(ignore_result=True)diff --git a/posthog/test/test_feature_flag_analytics.py b/posthog/test/test_feature_flag_analytics.pyindex 4d1946ea6591..4629a0b56cc6 100644--- a/posthog/test/test_feature_flag_analytics.py+++ b/posthog/test/test_feature_flag_analytics.py@@ -20,6 +20,7 @@ from django.core.cache import cachefrom django.test import SimpleTestCasefrom parameterized import parameterized+from redis.exceptions import ConnectionError as RedisConnectionErrorfrom posthog import redisfrom posthog.constants import FlagRequestType@@ -29,7 +30,10 @@ from posthog.helpers.dashboard_templates import (FEATURE_FLAG_ENRICHED_VIEW_INSIGHT_NAME,)from posthog.models.team.team import Team-from posthog.tasks.tasks import find_flags_with_enriched_analytics as find_flags_with_enriched_analytics_task+from posthog.tasks.tasks import (+ calculate_decide_usage,+ find_flags_with_enriched_analytics as find_flags_with_enriched_analytics_task,+)from products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.flag_analytics import (@@ -283,6 +287,7 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):)@patch("products.feature_flags.backend.flag_analytics.CACHE_BUCKET_SIZE", 10)+ @patch("products.feature_flags.backend.flag_analytics.TEAM_KEY_CHECK_BATCH_SIZE", 2)def test_efficient_querying_of_team_decide_usage_data(self):mock_capture = MagicMock()team_id = 3901@@ -297,6 +302,8 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):Team.objects.create(id=other_team_id, organization=self.organization, api_token=f"token:::{other_team_id}", uuid=other_team_uuid)+ local_evaluation_team_id = Team.objects.create(organization=self.organization).id+ remote_config_team_id = Team.objects.create(organization=self.organization).idwith time_machine.travel("2022-05-07 12:23:07", tick=False) as frozen_datetime:for _ in range(10):@@ -305,6 +312,8 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):for _ in range(7):# 7 requests for other teamincrement_request_count(other_team_id)+ increment_request_count(local_evaluation_team_id, request_type=FlagRequestType.LOCAL_EVALUATION)+ increment_request_count(remote_config_team_id, request_type=FlagRequestType.REMOTE_CONFIG)frozen_datetime.shift(datetime.timedelta(seconds=5))@@ -321,8 +330,17 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):# 5 requests in third bucketincrement_request_count(team_id)increment_request_count(other_team_id)-- with self.settings(DECIDE_BILLING_ANALYTICS_TOKEN="token"), snapshot_postgres_queries_context(self):+ increment_request_count(local_evaluation_team_id, request_type=FlagRequestType.LOCAL_EVALUATION)+ increment_request_count(remote_config_team_id, request_type=FlagRequestType.REMOTE_CONFIG)++ with (+ self.settings(DECIDE_BILLING_ANALYTICS_TOKEN="token"),+ snapshot_postgres_queries_context(self),+ patch(+ "products.feature_flags.backend.flag_analytics.capture_team_decide_usage",+ wraps=capture_team_decide_usage,+ ) as team_export,+ ):capture_usage_for_all_teams(mock_capture)mock_capture.capture.assert_any_call(@@ -350,7 +368,37 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):"token": "token",},)- assert mock_capture.capture.call_count == 2+ exported = {+ (call.kwargs["distinct_id"], call.kwargs["event"]) for call in mock_capture.capture.call_args_list+ }+ assert exported == {+ (team_id, "decide usage"),+ (other_team_id, "decide usage"),+ (local_evaluation_team_id, "local evaluation usage"),+ (remote_config_team_id, "remote config usage"),+ }+ assert mock_capture.capture.call_count == 4+ assert {call.args[1] for call in team_export.call_args_list} == {+ team_id,+ other_team_id,+ local_evaluation_team_id,+ remote_config_team_id,+ }++ def test_failed_key_check_exports_every_team_in_the_batch(self) -> None:+ other_team = Team.objects.create(organization=self.organization)+ client = MagicMock()+ client.pipeline.return_value.execute.side_effect = RedisConnectionError("Redis unavailable")++ with (+ patch("products.feature_flags.backend.flag_analytics.get_client", return_value=client),+ patch("products.feature_flags.backend.flag_analytics.capture_team_decide_usage") as team_export,+ patch("products.feature_flags.backend.flag_analytics.capture_exception") as capture_error,+ ):+ capture_usage_for_all_teams(MagicMock())++ assert {self.team.id, other_team.id} <= {call.args[1] for call in team_export.call_args_list}+ capture_error.assert_called_once()@pytest.mark.skip(reason="This works locally, but causes issues in CI because the frozen clock applies to threads as well in unrelated tests, causing timeouts."@@ -1353,3 +1401,16 @@ class TestEnrichedFlagKeyExprSQL(SimpleTestCase):):sql = _enriched_flag_key_expr_sql()assert sql == "ifNull(`mat_feature_flag`, '')"+++class TestCalculateDecideUsage(SimpleTestCase):+ @patch(+ "products.feature_flags.backend.flag_analytics.capture_usage_for_all_teams",+ side_effect=ConnectionError("redis"),+ )+ @patch("posthog.tasks.tasks.get_regional_ph_client")+ def test_shuts_down_the_client_when_the_export_fails(self, get_regional_ph_client, _capture):+ with pytest.raises(ConnectionError):+ calculate_decide_usage()++ get_regional_ph_client.return_value.shutdown.assert_called_once()diff --git a/products/feature_flags/backend/flag_analytics.py b/products/feature_flags/backend/flag_analytics.pyindex 3b840ca4b5b9..d55af90cf973 100644--- a/products/feature_flags/backend/flag_analytics.py+++ b/products/feature_flags/backend/flag_analytics.py@@ -33,6 +33,7 @@ logger = structlog.get_logger(__name__)REDIS_LOCK_TOKEN = "posthog:decide_analytics:lock"CACHE_BUCKET_SIZE = 60 * 2 # duration in seconds+TEAM_KEY_CHECK_BATCH_SIZE = 1000# Per-team counts of the local evaluation requests that Django served, held until the next usage report.# The legacy local evaluation counter also holds requests that the Rust definitions service counted@@ -87,6 +88,8 @@ _REQUEST_BUCKET_PREFIXES = {FlagRequestType.REMOTE_CONFIG: "remote_config_requests",}+EXPORTED_REQUEST_TYPES = (FlagRequestType.DECIDE, FlagRequestType.LOCAL_EVALUATION, FlagRequestType.REMOTE_CONFIG)+USAGE_EVENT_NAMES = {FlagRequestType.DECIDE: "decide usage",FlagRequestType.LOCAL_EVALUATION: "local evaluation usage",@@ -234,8 +237,23 @@ def _extract_sdk_breakdown_from_redis(def capture_usage_for_all_teams(ph_client: "Posthog") -> None:- for team in Team.objects.exclude(Q(organization__for_internal_metrics=True) | Q(is_demo=True)).only("id", "uuid"):- capture_team_decide_usage(ph_client, team.id, team.uuid)+ client = get_client()+ teams = Team.objects.exclude(Q(organization__for_internal_metrics=True) | Q(is_demo=True)).only("id", "uuid")+ # Most teams have no flag requests, so one pipelined key check per batch skips them in bulk.+ for batch in batched(teams, TEAM_KEY_CHECK_BATCH_SIZE, strict=False):+ pipe = client.pipeline(transaction=False)+ for team in batch:+ pipe.exists(*(get_team_request_key(team.id, request_type) for request_type in EXPORTED_REQUEST_TYPES))+ try:+ key_counts = pipe.execute()+ except redis.exceptions.RedisError as error:+ # Export the whole batch so that one failed check does not stop the pass.+ # The per-team export catches its own Redis errors.+ capture_exception(error)+ key_counts = [len(EXPORTED_REQUEST_TYPES)] * len(batch)+ for team, existing_keys in zip(batch, key_counts):+ if existing_keys:+ capture_team_decide_usage(ph_client, team.id, team.uuid)def _capture_team_usage_for_request_type(@@ -277,11 +295,7 @@ def capture_team_decide_usage(ph_client: "Posthog", team_id: int, team_uuid: strwith client.lock(f"{REDIS_LOCK_TOKEN}:{team_id}", timeout=60, blocking=False):billing_token = settings.DECIDE_BILLING_ANALYTICS_TOKEN- for request_type in (- FlagRequestType.DECIDE,- FlagRequestType.LOCAL_EVALUATION,- FlagRequestType.REMOTE_CONFIG,- ):+ for request_type in EXPORTED_REQUEST_TYPES:_capture_team_usage_for_request_type(ph_client, client, team_id, team_uuid, request_type, billing_token)except redis.exceptions.LockError: