:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): report django local evaluation usage (#118814)
Alex Mays5128b3f50ea3
diff --git a/docs/internal/feature-flags/billing.md b/docs/internal/feature-flags/billing.mdindex 14f76b4279e0..d377ebb75607 100644--- a/docs/internal/feature-flags/billing.md+++ b/docs/internal/feature-flags/billing.md@@ -116,6 +116,30 @@ The service extracts the SDK type from the request's user-agent header and incre| `posthog-flutter` | Flutter SDK || `other` | Unknown or unrecognized |+### Django flag list endpoint++The Django flag list endpoint counts each request made with a personal API key as a local evaluation request.+This applies to `/api/feature_flag/` and to `/api/projects/:project_id/feature_flags/`.+Browser-session and unauthenticated requests are not counted.++Each counted request increments two Redis hashes in one pipelined round trip:++- The legacy team counter, `posthog:local_evaluation_requests:{team_id}`, which the 30-minute aggregation reads.+- `posthog:django_local_evaluation_usage`, which holds one per-team count of the requests that Django served.++The second hash exists because the Rust definitions service also writes to the legacy team counter and reports its own usage records.++Every minute, `report_django_local_evaluation_usage_task` reads and deletes `posthog:django_local_evaluation_usage` in one Redis transaction.+It then sends one `feature_flag_local_evaluation_requests` record per team as the `feature-flags-django` producer.+The task sends at most 500 records in each request, because usage ingestion rejects a request above its batch limit.+The task runs on the `feature_flags_long_running` queue, so a stalled usage-ingestion service cannot delay the cache rebuilds on the `feature_flags` queue.++The workers on that queue need `USAGE_INGESTION_ADDR` with an explicit port, and an enabled `USAGE_INGESTION_REPORT_TEAMS` allowlist.+The Python gRPC client connects to port 443 when the address has no port, but the usage-ingestion Service listens on port 80.+The task discards counts for teams that the allowlist does not enable.+Delivery is best effort: the shared usage-ingestion client logs a failed send, and the task does not put the counts back.+This counter is in `RECORD_REALTIME_BLOCKERS`, so its `realtime` mode resolves to `both` and legacy totals stay authoritative during the comparison rollout.+### Redis pipeliningBoth the team-level and SDK-level counters are incremented in a single Redis round-trip using pipelining:diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.pyindex e0d3cb02d316..4cd608b7e228 100644--- a/posthog/tasks/scheduled.py+++ b/posthog/tasks/scheduled.py@@ -109,6 +109,7 @@ from products.feature_flags.backend.tasks import (feature_flags_local_eval_canary_task,refresh_expiring_flag_definitions_cache_entries,refresh_expiring_flags_cache_entries,+ report_django_local_evaluation_usage_task,sync_cross_region_flags_task,)from products.legal_documents.backend.facade.tasks import reconcile_pending_legal_documents@@ -655,6 +656,16 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:expires=30,)+ # Every minute, send the local evaluation requests that Django counted to usage ingestion.+ # The interval sets only how fresh the records are, because unsent counts wait in Redis.+ add_periodic_task_with_expiry(+ sender,+ crontab(minute="*"),+ report_django_local_evaluation_usage_task.s(),+ name="report django local evaluation usage",+ expires_seconds=60,+ )+add_periodic_task_with_expiry(sender,crontab(minute="*/5"),diff --git a/posthog/test/test_feature_flag_analytics.py b/posthog/test/test_feature_flag_analytics.pyindex 882f3f95e245..4d1946ea6591 100644--- a/posthog/test/test_feature_flag_analytics.py+++ b/posthog/test/test_feature_flag_analytics.py@@ -33,6 +33,7 @@ from posthog.tasks.tasks import find_flags_with_enriched_analytics as find_flagsfrom products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.flag_analytics import (+ DJANGO_LOCAL_EVALUATION_USAGE_KEY,SDK_LIBRARIES,_enriched_flag_key_expr_sql,_extract_sdk_breakdown_from_redis,@@ -44,6 +45,7 @@ from products.feature_flags.backend.flag_analytics import (get_evaluations_7d_by_team,get_team_request_library_key,increment_request_count,+ report_django_local_evaluation_usage,)from products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig@@ -919,6 +921,46 @@ class TestSdkBreakdown(BaseTest):)+class TestDjangoLocalEvaluationUsage(SimpleTestCase):+ def setUp(self) -> None:+ redis.get_client().delete(DJANGO_LOCAL_EVALUATION_USAGE_KEY)++ def _count(self, *team_ids: int) -> None:+ for team_id in team_ids:+ redis.get_client().hincrby(DJANGO_LOCAL_EVALUATION_USAGE_KEY, str(team_id), 1)++ def _report(self) -> list[list[tuple[int, int]]]:+ with (+ patch.dict("os.environ", {"USAGE_INGESTION_ADDR": "localhost:1", "USAGE_INGESTION_REPORT_TEAMS": "*"}),+ patch("posthog.usage_ingestion.client.grpc.insecure_channel"),+ patch("posthog.usage_ingestion.client.service_pb2_grpc.UsageIngestionStub") as stub,+ ):+ report_django_local_evaluation_usage()+ requests = [call.args[0] for call in stub.return_value.IngestBillingUsage.call_args_list]+ for request in requests:+ for record in request.records:+ self.assertEqual(+ (record.producer_id, record.usage_key, record.unit),+ ("feature-flags-django", "feature_flag_local_evaluation_requests", "requests"),+ )+ return [sorted((record.team_id, record.quantity) for record in request.records) for request in requests]++ def test_reports_each_count_once(self) -> None:+ self._count(1, 1, 1, 2)++ self.assertEqual(self._report(), [[(1, 3), (2, 1)]])++ self._count(1)+ self.assertEqual(self._report(), [[(1, 1)]])+ self.assertEqual(self._report(), [])++ @patch("products.feature_flags.backend.flag_analytics.USAGE_RECORDS_PER_REQUEST", 2)+ def test_splits_records_at_the_ingestion_batch_limit(self) -> None:+ self._count(1, 2, 3)++ self.assertEqual(sorted(len(request) for request in self._report()), [1, 2])++class TestEnrichedAnalytics(BaseTest):def test_find_flags_with_enriched_analytics(self):f1 = FeatureFlag.objects.create(diff --git a/posthog/usage_counters.py b/posthog/usage_counters.pyindex 1752bf04d839..0a818fa86832 100644--- a/posthog/usage_counters.py+++ b/posthog/usage_counters.py@@ -166,8 +166,8 @@ _CDP_DATE_ATTRIBUTION = ("A cutover must reconcile dispatches across those dates to avoid missing or double-counting them.")_LOCAL_EVALUATION_UNDERCOUNT = (- "Django's feature-flag list endpoint counts older SDK requests in Redis but emits no usage records. "- "Only the Rust definitions endpoint contributes to the record total."+ "Historical local-evaluation records omit requests handled by Django's feature-flag list endpoint. "+ "Validate both producers over a complete comparison window before using records as authoritative.")# Keep legacy totals authoritative until producer discrepancies are resolved or confirmed as intended billing behavior.RECORD_REALTIME_BLOCKERS = {diff --git a/products/feature_flags/backend/api/feature_flag.py b/products/feature_flags/backend/api/feature_flag.pyindex 196910c3c27d..a7faa7b53104 100644--- a/products/feature_flags/backend/api/feature_flag.py+++ b/products/feature_flags/backend/api/feature_flag.py@@ -147,7 +147,10 @@ from products.feature_flags.backend.facade.config import (from products.feature_flags.backend.facade.config_validation import ConfigValidationError, ValidationLimitsfrom products.feature_flags.backend.facade.references import InvalidIds, referencesfrom products.feature_flags.backend.filters_validation import collect_cross_field_violations, flatten_structural_errors-from products.feature_flags.backend.flag_analytics import increment_request_count+from products.feature_flags.backend.flag_analytics import (+ increment_local_evaluation_request_counts,+ increment_request_count,+)from products.feature_flags.backend.flag_limits import get_max_feature_flags_for_teamfrom products.feature_flags.backend.flag_status import (STALE_ACTIVE_PARAM_DESCRIPTION,@@ -4489,7 +4492,7 @@ class FeatureFlagViewSet(def list(self, request, *args, **kwargs):if isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication):# Add request for analytics only if request coming with personal API key authentication- increment_request_count(self.team.pk, 1, FlagRequestType.LOCAL_EVALUATION)+ increment_local_evaluation_request_counts(self.team.pk)response = super().list(request, *args, **kwargs)feature_flags_data = response.data.get("results", [])diff --git a/products/feature_flags/backend/api/test/test_feature_flag.py b/products/feature_flags/backend/api/test/test_feature_flag.pyindex 90b327de0819..2920f053ce21 100644--- a/products/feature_flags/backend/api/test/test_feature_flag.py+++ b/products/feature_flags/backend/api/test/test_feature_flag.py@@ -4895,6 +4895,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):# shouldn't add to local eval requestsself.assertEqual(client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"), {})+ self.assertIsNone(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)))self.client.logout()# `local_evaluation` is called by logged out clients!@@ -4908,6 +4909,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):response = self.client.get(f"/api/feature_flag/")self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)self.assertEqual(client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"), {})+ self.assertIsNone(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)))response = self.client.get(f"/api/feature_flag/?token={self.team.api_token}",@@ -4918,6 +4920,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"),{b"165192618": b"1"},)+ self.assertEqual(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)), b"1")for _ in range(4):response = self.client.get(@@ -4930,6 +4933,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"),{b"165192618": b"5"},)+ self.assertEqual(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)), b"5")@parameterized.expand([diff --git a/products/feature_flags/backend/flag_analytics.py b/products/feature_flags/backend/flag_analytics.pyindex b6f6af27cba5..3b840ca4b5b9 100644--- a/products/feature_flags/backend/flag_analytics.py+++ b/products/feature_flags/backend/flag_analytics.py@@ -1,7 +1,8 @@import timefrom datetime import datetime+from itertools import batchedfrom typing import TYPE_CHECKING-from uuid import UUID+from uuid import UUID, uuid4from django.conf import settingsfrom django.core.cache import cache@@ -19,6 +20,7 @@ from posthog.helpers.dashboard_templates import add_enriched_insights_to_featurefrom posthog.models import Teamfrom posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TABLEfrom posthog.redis import get_client, redis+from posthog.usage_ingestion.client import UsageRecord, report_usagefrom products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.facade.flags import get_flag_evaluations_read_mode@@ -32,6 +34,13 @@ logger = structlog.get_logger(__name__)REDIS_LOCK_TOKEN = "posthog:decide_analytics:lock"CACHE_BUCKET_SIZE = 60 * 2 # duration in seconds+# Per-team counts of the local evaluation requests that Django served, held until the next usage report.+# The legacy local evaluation counter also holds requests that the Rust definitions service counted+# and reported itself, so reporting from that counter would count those requests twice.+DJANGO_LOCAL_EVALUATION_USAGE_KEY = "posthog:django_local_evaluation_usage"+# usage-ingestion rejects a request with more records than USAGE_INGESTION_MAX_BATCH_SIZE, which defaults to 500.+USAGE_RECORDS_PER_REQUEST = 500+# SDK library names must match the Rust Library::as_str() values in# rust/feature-flags/src/handler/types.rsSDK_LIBRARIES = [@@ -113,6 +122,42 @@ def increment_request_count(capture_exception(error)+def increment_local_evaluation_request_counts(team_id: int) -> None:+ # One round trip for both counters, so a Redis stall delays the flag list request once.+ try:+ pipe = get_client().pipeline(transaction=False)+ pipe.hincrby(+ get_team_request_key(team_id, FlagRequestType.LOCAL_EVALUATION),+ str(int(time.time() / CACHE_BUCKET_SIZE)),+ 1,+ )+ pipe.hincrby(DJANGO_LOCAL_EVALUATION_USAGE_KEY, str(team_id), 1)+ pipe.execute()+ except Exception as error:+ capture_exception(error)+++def report_django_local_evaluation_usage() -> None:+ # One transaction reads and clears the counts, so a request counted after it waits for the next report.+ pipe = get_client().pipeline(transaction=True)+ pipe.hgetall(DJANGO_LOCAL_EVALUATION_USAGE_KEY)+ pipe.delete(DJANGO_LOCAL_EVALUATION_USAGE_KEY)+ counts, _ = pipe.execute()+ records = [+ UsageRecord(+ record_id=str(uuid4()),+ producer_id="feature-flags-django",+ team_id=int(team_id),+ usage_key="feature_flag_local_evaluation_requests",+ unit="requests",+ quantity=int(count),+ )+ for team_id, count in counts.items()+ ]+ for chunk in batched(records, USAGE_RECORDS_PER_REQUEST, strict=False):+ report_usage(chunk, site="django_flag_definitions")++def _extract_total_count_for_key_from_redis_hash(client: redis.Redis, key: str) -> tuple[int, int, int]:total_count = 0existing_values = client.hgetall(key)diff --git a/products/feature_flags/backend/tasks.py b/products/feature_flags/backend/tasks.pyindex 188f7f1dfd9e..ef465c3fe7ea 100644--- a/products/feature_flags/backend/tasks.py+++ b/products/feature_flags/backend/tasks.py@@ -24,6 +24,7 @@ from posthog.tasks.utils import CeleryQueue, PushGatewayTaskfrom products.feature_flags.backend.canary import run_local_eval_canaryfrom products.feature_flags.backend.cross_region_flag_sync import sync_cross_region_flags+from products.feature_flags.backend.flag_analytics import report_django_local_evaluation_usagefrom products.feature_flags.backend.flags_cache import (cleanup_stale_expiry_tracking,clear_flags_cache,@@ -87,6 +88,16 @@ def sync_cross_region_flags_task() -> None:sync_cross_region_flags()+@shared_task(ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS_LONG_RUNNING.value)+def report_django_local_evaluation_usage_task() -> None:+ """Celery entrypoint for report_django_local_evaluation_usage.++ On the long-running queue for the same reason as sync_cross_region_flags_task:+ the usage-ingestion call blocks for its retries when that service stalls.+ """+ report_django_local_evaluation_usage()++# Pinned: products.cohorts dispatches this by name (a static import would close a product-dependency# cycle), so a module move must not silently rename the registration out from under that caller.@shared_task(