Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): report django local evaluation usage (#118814)

Alex Mays5128b3f50ea3
diff --git a/docs/internal/feature-flags/billing.md b/docs/internal/feature-flags/billing.md
index 14f76b4279e0..d377ebb75607 100644
--- a/docs/internal/feature-flags/billing.md
+++ b/docs/internal/feature-flags/billing.md
@@ -116,6 +116,30 @@ The service extracts the SDK type from the request's user-agent header and incre
| `posthog-flutter` | Flutter SDK |
| `other` | Unknown or unrecognized |
+### Django flag list endpoint
+
+The Django flag list endpoint counts each request made with a personal API key as a local evaluation request.
+This applies to `/api/feature_flag/` and to `/api/projects/:project_id/feature_flags/`.
+Browser-session and unauthenticated requests are not counted.
+
+Each counted request increments two Redis hashes in one pipelined round trip:
+
+- The legacy team counter, `posthog:local_evaluation_requests:{team_id}`, which the 30-minute aggregation reads.
+- `posthog:django_local_evaluation_usage`, which holds one per-team count of the requests that Django served.
+
+The second hash exists because the Rust definitions service also writes to the legacy team counter and reports its own usage records.
+
+Every minute, `report_django_local_evaluation_usage_task` reads and deletes `posthog:django_local_evaluation_usage` in one Redis transaction.
+It then sends one `feature_flag_local_evaluation_requests` record per team as the `feature-flags-django` producer.
+The task sends at most 500 records in each request, because usage ingestion rejects a request above its batch limit.
+The task runs on the `feature_flags_long_running` queue, so a stalled usage-ingestion service cannot delay the cache rebuilds on the `feature_flags` queue.
+
+The workers on that queue need `USAGE_INGESTION_ADDR` with an explicit port, and an enabled `USAGE_INGESTION_REPORT_TEAMS` allowlist.
+The Python gRPC client connects to port 443 when the address has no port, but the usage-ingestion Service listens on port 80.
+The task discards counts for teams that the allowlist does not enable.
+Delivery is best effort: the shared usage-ingestion client logs a failed send, and the task does not put the counts back.
+This counter is in `RECORD_REALTIME_BLOCKERS`, so its `realtime` mode resolves to `both` and legacy totals stay authoritative during the comparison rollout.
+
### Redis pipelining
Both the team-level and SDK-level counters are incremented in a single Redis round-trip using pipelining:
diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.py
index e0d3cb02d316..4cd608b7e228 100644
--- a/posthog/tasks/scheduled.py
+++ b/posthog/tasks/scheduled.py
@@ -109,6 +109,7 @@ from products.feature_flags.backend.tasks import (
feature_flags_local_eval_canary_task,
refresh_expiring_flag_definitions_cache_entries,
refresh_expiring_flags_cache_entries,
+ report_django_local_evaluation_usage_task,
sync_cross_region_flags_task,
)
from products.legal_documents.backend.facade.tasks import reconcile_pending_legal_documents
@@ -655,6 +656,16 @@ def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:
expires=30,
)
+ # Every minute, send the local evaluation requests that Django counted to usage ingestion.
+ # The interval sets only how fresh the records are, because unsent counts wait in Redis.
+ add_periodic_task_with_expiry(
+ sender,
+ crontab(minute="*"),
+ report_django_local_evaluation_usage_task.s(),
+ name="report django local evaluation usage",
+ expires_seconds=60,
+ )
+
add_periodic_task_with_expiry(
sender,
crontab(minute="*/5"),
diff --git a/posthog/test/test_feature_flag_analytics.py b/posthog/test/test_feature_flag_analytics.py
index 882f3f95e245..4d1946ea6591 100644
--- a/posthog/test/test_feature_flag_analytics.py
+++ b/posthog/test/test_feature_flag_analytics.py
@@ -33,6 +33,7 @@ from posthog.tasks.tasks import find_flags_with_enriched_analytics as find_flags
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.flag_analytics import (
+ DJANGO_LOCAL_EVALUATION_USAGE_KEY,
SDK_LIBRARIES,
_enriched_flag_key_expr_sql,
_extract_sdk_breakdown_from_redis,
@@ -44,6 +45,7 @@ from products.feature_flags.backend.flag_analytics import (
get_evaluations_7d_by_team,
get_team_request_library_key,
increment_request_count,
+ report_django_local_evaluation_usage,
)
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -919,6 +921,46 @@ class TestSdkBreakdown(BaseTest):
)
+class TestDjangoLocalEvaluationUsage(SimpleTestCase):
+ def setUp(self) -> None:
+ redis.get_client().delete(DJANGO_LOCAL_EVALUATION_USAGE_KEY)
+
+ def _count(self, *team_ids: int) -> None:
+ for team_id in team_ids:
+ redis.get_client().hincrby(DJANGO_LOCAL_EVALUATION_USAGE_KEY, str(team_id), 1)
+
+ def _report(self) -> list[list[tuple[int, int]]]:
+ with (
+ patch.dict("os.environ", {"USAGE_INGESTION_ADDR": "localhost:1", "USAGE_INGESTION_REPORT_TEAMS": "*"}),
+ patch("posthog.usage_ingestion.client.grpc.insecure_channel"),
+ patch("posthog.usage_ingestion.client.service_pb2_grpc.UsageIngestionStub") as stub,
+ ):
+ report_django_local_evaluation_usage()
+ requests = [call.args[0] for call in stub.return_value.IngestBillingUsage.call_args_list]
+ for request in requests:
+ for record in request.records:
+ self.assertEqual(
+ (record.producer_id, record.usage_key, record.unit),
+ ("feature-flags-django", "feature_flag_local_evaluation_requests", "requests"),
+ )
+ return [sorted((record.team_id, record.quantity) for record in request.records) for request in requests]
+
+ def test_reports_each_count_once(self) -> None:
+ self._count(1, 1, 1, 2)
+
+ self.assertEqual(self._report(), [[(1, 3), (2, 1)]])
+
+ self._count(1)
+ self.assertEqual(self._report(), [[(1, 1)]])
+ self.assertEqual(self._report(), [])
+
+ @patch("products.feature_flags.backend.flag_analytics.USAGE_RECORDS_PER_REQUEST", 2)
+ def test_splits_records_at_the_ingestion_batch_limit(self) -> None:
+ self._count(1, 2, 3)
+
+ self.assertEqual(sorted(len(request) for request in self._report()), [1, 2])
+
+
class TestEnrichedAnalytics(BaseTest):
def test_find_flags_with_enriched_analytics(self):
f1 = FeatureFlag.objects.create(
diff --git a/posthog/usage_counters.py b/posthog/usage_counters.py
index 1752bf04d839..0a818fa86832 100644
--- a/posthog/usage_counters.py
+++ b/posthog/usage_counters.py
@@ -166,8 +166,8 @@ _CDP_DATE_ATTRIBUTION = (
"A cutover must reconcile dispatches across those dates to avoid missing or double-counting them."
)
_LOCAL_EVALUATION_UNDERCOUNT = (
- "Django's feature-flag list endpoint counts older SDK requests in Redis but emits no usage records. "
- "Only the Rust definitions endpoint contributes to the record total."
+ "Historical local-evaluation records omit requests handled by Django's feature-flag list endpoint. "
+ "Validate both producers over a complete comparison window before using records as authoritative."
)
# Keep legacy totals authoritative until producer discrepancies are resolved or confirmed as intended billing behavior.
RECORD_REALTIME_BLOCKERS = {
diff --git a/products/feature_flags/backend/api/feature_flag.py b/products/feature_flags/backend/api/feature_flag.py
index 196910c3c27d..a7faa7b53104 100644
--- a/products/feature_flags/backend/api/feature_flag.py
+++ b/products/feature_flags/backend/api/feature_flag.py
@@ -147,7 +147,10 @@ from products.feature_flags.backend.facade.config import (
from products.feature_flags.backend.facade.config_validation import ConfigValidationError, ValidationLimits
from products.feature_flags.backend.facade.references import InvalidIds, references
from products.feature_flags.backend.filters_validation import collect_cross_field_violations, flatten_structural_errors
-from products.feature_flags.backend.flag_analytics import increment_request_count
+from products.feature_flags.backend.flag_analytics import (
+ increment_local_evaluation_request_counts,
+ increment_request_count,
+)
from products.feature_flags.backend.flag_limits import get_max_feature_flags_for_team
from products.feature_flags.backend.flag_status import (
STALE_ACTIVE_PARAM_DESCRIPTION,
@@ -4489,7 +4492,7 @@ class FeatureFlagViewSet(
def list(self, request, *args, **kwargs):
if isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication):
# Add request for analytics only if request coming with personal API key authentication
- increment_request_count(self.team.pk, 1, FlagRequestType.LOCAL_EVALUATION)
+ increment_local_evaluation_request_counts(self.team.pk)
response = super().list(request, *args, **kwargs)
feature_flags_data = response.data.get("results", [])
diff --git a/products/feature_flags/backend/api/test/test_feature_flag.py b/products/feature_flags/backend/api/test/test_feature_flag.py
index 90b327de0819..2920f053ce21 100644
--- a/products/feature_flags/backend/api/test/test_feature_flag.py
+++ b/products/feature_flags/backend/api/test/test_feature_flag.py
@@ -4895,6 +4895,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):
# shouldn't add to local eval requests
self.assertEqual(client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"), {})
+ self.assertIsNone(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)))
self.client.logout()
# `local_evaluation` is called by logged out clients!
@@ -4908,6 +4909,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):
response = self.client.get(f"/api/feature_flag/")
self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
self.assertEqual(client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"), {})
+ self.assertIsNone(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)))
response = self.client.get(
f"/api/feature_flag/?token={self.team.api_token}",
@@ -4918,6 +4920,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):
client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"),
{b"165192618": b"1"},
)
+ self.assertEqual(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)), b"1")
for _ in range(4):
response = self.client.get(
@@ -4930,6 +4933,7 @@ class TestFeatureFlag(APIBaseTest, ClickhouseTestMixin):
client.hgetall(f"posthog:local_evaluation_requests:{self.team.pk}"),
{b"165192618": b"5"},
)
+ self.assertEqual(client.hget("posthog:django_local_evaluation_usage", str(self.team.pk)), b"5")
@parameterized.expand(
[
diff --git a/products/feature_flags/backend/flag_analytics.py b/products/feature_flags/backend/flag_analytics.py
index b6f6af27cba5..3b840ca4b5b9 100644
--- a/products/feature_flags/backend/flag_analytics.py
+++ b/products/feature_flags/backend/flag_analytics.py
@@ -1,7 +1,8 @@
import time
from datetime import datetime
+from itertools import batched
from typing import TYPE_CHECKING
-from uuid import UUID
+from uuid import UUID, uuid4
from django.conf import settings
from django.core.cache import cache
@@ -19,6 +20,7 @@ from posthog.helpers.dashboard_templates import add_enriched_insights_to_feature
from posthog.models import Team
from posthog.models.flag_evaluations.sql import FLAG_EVALUATIONS_TABLE
from posthog.redis import get_client, redis
+from posthog.usage_ingestion.client import UsageRecord, report_usage
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.facade.flags import get_flag_evaluations_read_mode
@@ -32,6 +34,13 @@ logger = structlog.get_logger(__name__)
REDIS_LOCK_TOKEN = "posthog:decide_analytics:lock"
CACHE_BUCKET_SIZE = 60 * 2 # duration in seconds
+# Per-team counts of the local evaluation requests that Django served, held until the next usage report.
+# The legacy local evaluation counter also holds requests that the Rust definitions service counted
+# and reported itself, so reporting from that counter would count those requests twice.
+DJANGO_LOCAL_EVALUATION_USAGE_KEY = "posthog:django_local_evaluation_usage"
+# usage-ingestion rejects a request with more records than USAGE_INGESTION_MAX_BATCH_SIZE, which defaults to 500.
+USAGE_RECORDS_PER_REQUEST = 500
+
# SDK library names must match the Rust Library::as_str() values in
# rust/feature-flags/src/handler/types.rs
SDK_LIBRARIES = [
@@ -113,6 +122,42 @@ def increment_request_count(
capture_exception(error)
+def increment_local_evaluation_request_counts(team_id: int) -> None:
+ # One round trip for both counters, so a Redis stall delays the flag list request once.
+ try:
+ pipe = get_client().pipeline(transaction=False)
+ pipe.hincrby(
+ get_team_request_key(team_id, FlagRequestType.LOCAL_EVALUATION),
+ str(int(time.time() / CACHE_BUCKET_SIZE)),
+ 1,
+ )
+ pipe.hincrby(DJANGO_LOCAL_EVALUATION_USAGE_KEY, str(team_id), 1)
+ pipe.execute()
+ except Exception as error:
+ capture_exception(error)
+
+
+def report_django_local_evaluation_usage() -> None:
+ # One transaction reads and clears the counts, so a request counted after it waits for the next report.
+ pipe = get_client().pipeline(transaction=True)
+ pipe.hgetall(DJANGO_LOCAL_EVALUATION_USAGE_KEY)
+ pipe.delete(DJANGO_LOCAL_EVALUATION_USAGE_KEY)
+ counts, _ = pipe.execute()
+ records = [
+ UsageRecord(
+ record_id=str(uuid4()),
+ producer_id="feature-flags-django",
+ team_id=int(team_id),
+ usage_key="feature_flag_local_evaluation_requests",
+ unit="requests",
+ quantity=int(count),
+ )
+ for team_id, count in counts.items()
+ ]
+ for chunk in batched(records, USAGE_RECORDS_PER_REQUEST, strict=False):
+ report_usage(chunk, site="django_flag_definitions")
+
+
def _extract_total_count_for_key_from_redis_hash(client: redis.Redis, key: str) -> tuple[int, int, int]:
total_count = 0
existing_values = client.hgetall(key)
diff --git a/products/feature_flags/backend/tasks.py b/products/feature_flags/backend/tasks.py
index 188f7f1dfd9e..ef465c3fe7ea 100644
--- a/products/feature_flags/backend/tasks.py
+++ b/products/feature_flags/backend/tasks.py
@@ -24,6 +24,7 @@ from posthog.tasks.utils import CeleryQueue, PushGatewayTask
from products.feature_flags.backend.canary import run_local_eval_canary
from products.feature_flags.backend.cross_region_flag_sync import sync_cross_region_flags
+from products.feature_flags.backend.flag_analytics import report_django_local_evaluation_usage
from products.feature_flags.backend.flags_cache import (
cleanup_stale_expiry_tracking,
clear_flags_cache,
@@ -87,6 +88,16 @@ def sync_cross_region_flags_task() -> None:
sync_cross_region_flags()
+@shared_task(ignore_result=True, queue=CeleryQueue.FEATURE_FLAGS_LONG_RUNNING.value)
+def report_django_local_evaluation_usage_task() -> None:
+ """Celery entrypoint for report_django_local_evaluation_usage.
+
+ On the long-running queue for the same reason as sync_cross_region_flags_task:
+ the usage-ingestion call blocks for its retries when that service stalls.
+ """
+ report_django_local_evaluation_usage()
+
+
# Pinned: products.cohorts dispatches this by name (a static import would close a product-dependency
# cycle), so a module move must not silently rename the registration out from under that caller.
@shared_task(