Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): skip teams without flag requests in the usage export (#122257)

Alex Mayse0e446e19e0b
diff --git a/docs/internal/feature-flags/billing.md b/docs/internal/feature-flags/billing.md
index 0c1d264aa05c..686318daff4f 100644
--- a/docs/internal/feature-flags/billing.md
+++ b/docs/internal/feature-flags/billing.md
@@ -185,7 +185,13 @@ sender.add_periodic_task(
### Aggregation process
-For each team (excluding internal metrics and demo teams):
+The task reads the teams 1,000 at a time, excluding internal metrics and demo teams.
+For each batch, one pipelined `EXISTS` call finds the teams that have a team key for any request type.
+The task skips the teams with no team key.
+Those teams also have no SDK keys, because the service writes the SDK key in the same pipeline as the team key.
+When the `EXISTS` call fails, the task reports the error and exports every team in that batch.
+
+For each team with a team key:
1. **Acquire distributed lock**: `posthog:decide_analytics:lock:{team_id}` with 60-second timeout
2. **Extract counters**: Read all time buckets except the current one (still being filled)
diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.py
index 9b348956c82c..8c9fee6d5da5 100644
--- a/posthog/tasks/tasks.py
+++ b/posthog/tasks/tasks.py
@@ -1032,8 +1032,10 @@ def calculate_decide_usage() -> None:
ph_client = get_regional_ph_client()
if ph_client:
- capture_decide_usage_for_all_teams(ph_client)
- ph_client.shutdown()
+ try:
+ capture_decide_usage_for_all_teams(ph_client)
+ finally:
+ ph_client.shutdown()
@shared_task(ignore_result=True)
diff --git a/posthog/test/test_feature_flag_analytics.py b/posthog/test/test_feature_flag_analytics.py
index 4d1946ea6591..4629a0b56cc6 100644
--- a/posthog/test/test_feature_flag_analytics.py
+++ b/posthog/test/test_feature_flag_analytics.py
@@ -20,6 +20,7 @@ from django.core.cache import cache
from django.test import SimpleTestCase
from parameterized import parameterized
+from redis.exceptions import ConnectionError as RedisConnectionError
from posthog import redis
from posthog.constants import FlagRequestType
@@ -29,7 +30,10 @@ from posthog.helpers.dashboard_templates import (
FEATURE_FLAG_ENRICHED_VIEW_INSIGHT_NAME,
)
from posthog.models.team.team import Team
-from posthog.tasks.tasks import find_flags_with_enriched_analytics as find_flags_with_enriched_analytics_task
+from posthog.tasks.tasks import (
+ calculate_decide_usage,
+ find_flags_with_enriched_analytics as find_flags_with_enriched_analytics_task,
+)
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.flag_analytics import (
@@ -283,6 +287,7 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):
)
@patch("products.feature_flags.backend.flag_analytics.CACHE_BUCKET_SIZE", 10)
+ @patch("products.feature_flags.backend.flag_analytics.TEAM_KEY_CHECK_BATCH_SIZE", 2)
def test_efficient_querying_of_team_decide_usage_data(self):
mock_capture = MagicMock()
team_id = 3901
@@ -297,6 +302,8 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):
Team.objects.create(
id=other_team_id, organization=self.organization, api_token=f"token:::{other_team_id}", uuid=other_team_uuid
)
+ local_evaluation_team_id = Team.objects.create(organization=self.organization).id
+ remote_config_team_id = Team.objects.create(organization=self.organization).id
with time_machine.travel("2022-05-07 12:23:07", tick=False) as frozen_datetime:
for _ in range(10):
@@ -305,6 +312,8 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):
for _ in range(7):
# 7 requests for other team
increment_request_count(other_team_id)
+ increment_request_count(local_evaluation_team_id, request_type=FlagRequestType.LOCAL_EVALUATION)
+ increment_request_count(remote_config_team_id, request_type=FlagRequestType.REMOTE_CONFIG)
frozen_datetime.shift(datetime.timedelta(seconds=5))
@@ -321,8 +330,17 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):
# 5 requests in third bucket
increment_request_count(team_id)
increment_request_count(other_team_id)
-
- with self.settings(DECIDE_BILLING_ANALYTICS_TOKEN="token"), snapshot_postgres_queries_context(self):
+ increment_request_count(local_evaluation_team_id, request_type=FlagRequestType.LOCAL_EVALUATION)
+ increment_request_count(remote_config_team_id, request_type=FlagRequestType.REMOTE_CONFIG)
+
+ with (
+ self.settings(DECIDE_BILLING_ANALYTICS_TOKEN="token"),
+ snapshot_postgres_queries_context(self),
+ patch(
+ "products.feature_flags.backend.flag_analytics.capture_team_decide_usage",
+ wraps=capture_team_decide_usage,
+ ) as team_export,
+ ):
capture_usage_for_all_teams(mock_capture)
mock_capture.capture.assert_any_call(
@@ -350,7 +368,37 @@ class TestFeatureFlagAnalytics(BaseTest, QueryMatchingTest):
"token": "token",
},
)
- assert mock_capture.capture.call_count == 2
+ exported = {
+ (call.kwargs["distinct_id"], call.kwargs["event"]) for call in mock_capture.capture.call_args_list
+ }
+ assert exported == {
+ (team_id, "decide usage"),
+ (other_team_id, "decide usage"),
+ (local_evaluation_team_id, "local evaluation usage"),
+ (remote_config_team_id, "remote config usage"),
+ }
+ assert mock_capture.capture.call_count == 4
+ assert {call.args[1] for call in team_export.call_args_list} == {
+ team_id,
+ other_team_id,
+ local_evaluation_team_id,
+ remote_config_team_id,
+ }
+
+ def test_failed_key_check_exports_every_team_in_the_batch(self) -> None:
+ other_team = Team.objects.create(organization=self.organization)
+ client = MagicMock()
+ client.pipeline.return_value.execute.side_effect = RedisConnectionError("Redis unavailable")
+
+ with (
+ patch("products.feature_flags.backend.flag_analytics.get_client", return_value=client),
+ patch("products.feature_flags.backend.flag_analytics.capture_team_decide_usage") as team_export,
+ patch("products.feature_flags.backend.flag_analytics.capture_exception") as capture_error,
+ ):
+ capture_usage_for_all_teams(MagicMock())
+
+ assert {self.team.id, other_team.id} <= {call.args[1] for call in team_export.call_args_list}
+ capture_error.assert_called_once()
@pytest.mark.skip(
reason="This works locally, but causes issues in CI because the frozen clock applies to threads as well in unrelated tests, causing timeouts."
@@ -1353,3 +1401,16 @@ class TestEnrichedFlagKeyExprSQL(SimpleTestCase):
):
sql = _enriched_flag_key_expr_sql()
assert sql == "ifNull(`mat_feature_flag`, '')"
+
+
+class TestCalculateDecideUsage(SimpleTestCase):
+ @patch(
+ "products.feature_flags.backend.flag_analytics.capture_usage_for_all_teams",
+ side_effect=ConnectionError("redis"),
+ )
+ @patch("posthog.tasks.tasks.get_regional_ph_client")
+ def test_shuts_down_the_client_when_the_export_fails(self, get_regional_ph_client, _capture):
+ with pytest.raises(ConnectionError):
+ calculate_decide_usage()
+
+ get_regional_ph_client.return_value.shutdown.assert_called_once()
diff --git a/products/feature_flags/backend/flag_analytics.py b/products/feature_flags/backend/flag_analytics.py
index 3b840ca4b5b9..d55af90cf973 100644
--- a/products/feature_flags/backend/flag_analytics.py
+++ b/products/feature_flags/backend/flag_analytics.py
@@ -33,6 +33,7 @@ logger = structlog.get_logger(__name__)
REDIS_LOCK_TOKEN = "posthog:decide_analytics:lock"
CACHE_BUCKET_SIZE = 60 * 2 # duration in seconds
+TEAM_KEY_CHECK_BATCH_SIZE = 1000
# Per-team counts of the local evaluation requests that Django served, held until the next usage report.
# The legacy local evaluation counter also holds requests that the Rust definitions service counted
@@ -87,6 +88,8 @@ _REQUEST_BUCKET_PREFIXES = {
FlagRequestType.REMOTE_CONFIG: "remote_config_requests",
}
+EXPORTED_REQUEST_TYPES = (FlagRequestType.DECIDE, FlagRequestType.LOCAL_EVALUATION, FlagRequestType.REMOTE_CONFIG)
+
USAGE_EVENT_NAMES = {
FlagRequestType.DECIDE: "decide usage",
FlagRequestType.LOCAL_EVALUATION: "local evaluation usage",
@@ -234,8 +237,23 @@ def _extract_sdk_breakdown_from_redis(
def capture_usage_for_all_teams(ph_client: "Posthog") -> None:
- for team in Team.objects.exclude(Q(organization__for_internal_metrics=True) | Q(is_demo=True)).only("id", "uuid"):
- capture_team_decide_usage(ph_client, team.id, team.uuid)
+ client = get_client()
+ teams = Team.objects.exclude(Q(organization__for_internal_metrics=True) | Q(is_demo=True)).only("id", "uuid")
+ # Most teams have no flag requests, so one pipelined key check per batch skips them in bulk.
+ for batch in batched(teams, TEAM_KEY_CHECK_BATCH_SIZE, strict=False):
+ pipe = client.pipeline(transaction=False)
+ for team in batch:
+ pipe.exists(*(get_team_request_key(team.id, request_type) for request_type in EXPORTED_REQUEST_TYPES))
+ try:
+ key_counts = pipe.execute()
+ except redis.exceptions.RedisError as error:
+ # Export the whole batch so that one failed check does not stop the pass.
+ # The per-team export catches its own Redis errors.
+ capture_exception(error)
+ key_counts = [len(EXPORTED_REQUEST_TYPES)] * len(batch)
+ for team, existing_keys in zip(batch, key_counts):
+ if existing_keys:
+ capture_team_decide_usage(ph_client, team.id, team.uuid)
def _capture_team_usage_for_request_type(
@@ -277,11 +295,7 @@ def capture_team_decide_usage(ph_client: "Posthog", team_id: int, team_uuid: str
with client.lock(f"{REDIS_LOCK_TOKEN}:{team_id}", timeout=60, blocking=False):
billing_token = settings.DECIDE_BILLING_ANALYTICS_TOKEN
- for request_type in (
- FlagRequestType.DECIDE,
- FlagRequestType.LOCAL_EVALUATION,
- FlagRequestType.REMOTE_CONFIG,
- ):
+ for request_type in EXPORTED_REQUEST_TYPES:
_capture_team_usage_for_request_type(ph_client, client, team_id, team_uuid, request_type, billing_token)
except redis.exceptions.LockError: