:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(endpoints): count 429 refusals by reason and send a per-team event (#121229)
Jovan Sakovicf208f806fd03
diff --git a/products/endpoints/backend/facade/api.py b/products/endpoints/backend/facade/api.pyindex 43be2c60f9e2..c334e771ccc3 100644--- a/products/endpoints/backend/facade/api.py+++ b/products/endpoints/backend/facade/api.py@@ -29,6 +29,7 @@ from datetime import datetimefrom typing import TYPE_CHECKINGfrom products.endpoints.backend import rate_limit+from products.endpoints.backend.logic.rate_limit_signal import EndpointRateLimitReason, record_endpoint_rate_limitedfrom products.endpoints.backend.models import Endpoint, EndpointVersionfrom . import contracts@@ -154,6 +155,16 @@ def is_materialized_request(team_id: int, endpoint_name: str, version: int | Nonreturn rate_limit.check_materialized_request(team_id, endpoint_name, version, request_data)+def record_throttled_request(+ team_id: int | None, endpoint_name: str | None, version: int | None, *, materialized: bool, scope: str+) -> None:+ """Record a run request refused by a request-rate throttle."""+ reason = EndpointRateLimitReason.MATERIALIZED_RATE if materialized else EndpointRateLimitReason.INLINE_RATE+ record_endpoint_rate_limited(+ team_id=team_id, endpoint_name=endpoint_name, version=version, reason=reason, scope=scope+ )++__all__ = ["REWRITE_CONTRACT","EndpointCrudService",@@ -168,6 +179,7 @@ __all__ = ["is_materialization_ready","is_materialized_request","list_endpoints",+ "record_throttled_request","live_materialization_conditions_source","materialization_fix_enabled","suggest_materialization_fix",diff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.pyindex f5614d29a00f..5021bf37c9be 100644--- a/products/endpoints/backend/logic/execution.py+++ b/products/endpoints/backend/logic/execution.py@@ -76,6 +76,7 @@ from products.endpoints.backend.exceptions import EndpointAtCapacity, EndpointQufrom products.endpoints.backend.insight_transformers import MaterializedSeriesMismatchErrorfrom products.endpoints.backend.logic.materialized_serving import serving_state_for_versionfrom products.endpoints.backend.logic.pagination import EndpointPagination+from products.endpoints.backend.logic.rate_limit_signal import EndpointRateLimitReason, record_endpoint_rate_limitedfrom products.endpoints.backend.logic.strategies import EndpointQueryStrategy, strategy_forfrom products.endpoints.backend.logic.validation import NATIVE_CLUSTER_UNSUPPORTEDfrom products.endpoints.backend.logs import build_execution_message, log_endpoint_execution@@ -637,10 +638,24 @@ class EndpointExecutionService(PydanticModelMixin):raise ValidationError("Query resolution failed: unable to resolve table or field references.")except ConcurrencyLimitExceeded:ENDPOINT_CONCURRENCY_REJECTED_TOTAL.labels(team_id=str(self.team.pk)).inc()+ record_endpoint_rate_limited(+ team_id=self.team.pk,+ endpoint_name=endpoint.name,+ version=version_obj.version,+ reason=EndpointRateLimitReason.CONCURRENCY,+ scope=EndpointRateLimitReason.CONCURRENCY.value,+ )raise Throttled(detail="Too many concurrent requests. Please try again later.")except APIQueriesBudgetExceeded:# The platform refused this query on purpose, so it is not an endpoint fault. Leave the# execution counter alone: the budget path counts its own refusals.+ record_endpoint_rate_limited(+ team_id=self.team.pk,+ endpoint_name=endpoint.name,+ version=version_obj.version,+ reason=EndpointRateLimitReason.READ_BUDGET,+ scope=EndpointRateLimitReason.READ_BUDGET.value,+ )raiseexcept tuple(_QUERY_PERFORMANCE_ERRORS) as e:execution_status = "query_performance"diff --git a/products/endpoints/backend/logic/rate_limit_signal.py b/products/endpoints/backend/logic/rate_limit_signal.pynew file mode 100644index 000000000000..69e7b7426283--- /dev/null+++ b/products/endpoints/backend/logic/rate_limit_signal.py@@ -0,0 +1,67 @@+"""Signals for endpoint run requests refused with a 429.++Every refusal increments ``posthog_endpoint_rate_limited_total`` (fleet view, no team label).+A ``endpoint rate limited`` analytics event carries the team, so refusals can be grouped+per team in the warehouse. The event is sent at most once per (team, endpoint, reason)+per window, so a team retrying in a tight loop does not flood the event stream.+"""++from enum import StrEnum++from django.core.cache import cache++import structlog+import posthoganalytics++from products.endpoints.backend.metrics import ENDPOINT_RATE_LIMITED_TOTAL++logger = structlog.get_logger(__name__)++RATE_LIMITED_EVENT = "endpoint rate limited"+RATE_LIMITED_EVENT_WINDOW_SECONDS = 60+_DEDUP_CACHE_KEY = "endpoint_rate_limited_event:{team_id}:{endpoint_name}:{reason}"+++class EndpointRateLimitReason(StrEnum):+ MATERIALIZED_RATE = "materialized_rate"+ INLINE_RATE = "inline_rate"+ CONCURRENCY = "concurrency"+ READ_BUDGET = "read_budget"+++def record_endpoint_rate_limited(+ *,+ team_id: int | None,+ endpoint_name: str | None,+ version: int | None,+ reason: EndpointRateLimitReason,+ scope: str,+) -> None:+ """Count a 429 and send the deduplicated per-team event. Never raises: this runs on the request path."""+ try:+ ENDPOINT_RATE_LIMITED_TOTAL.labels(scope=scope, reason=reason.value).inc()+ except Exception:+ logger.warning("endpoint_rate_limited_metric_failed", exc_info=True)++ if team_id is None or not endpoint_name:+ return++ try:+ dedup_key = _DEDUP_CACHE_KEY.format(team_id=team_id, endpoint_name=endpoint_name, reason=reason.value)+ if not cache.add(dedup_key, 1, timeout=RATE_LIMITED_EVENT_WINDOW_SECONDS):+ return+ # The SDK only enqueues here; a background thread delivers the batch.+ posthoganalytics.capture(+ distinct_id=f"team_{team_id}",+ event=RATE_LIMITED_EVENT,+ properties={+ "team_id": team_id,+ "endpoint_name": endpoint_name,+ "endpoint_version": version,+ "reason": reason.value,+ "scope": scope,+ "$process_person_profile": False,+ },+ )+ except Exception:+ logger.warning("endpoint_rate_limited_event_failed", exc_info=True)diff --git a/products/endpoints/backend/metrics.py b/products/endpoints/backend/metrics.pyindex 47a2bb0dafcf..74d35cef2bfb 100644--- a/products/endpoints/backend/metrics.py+++ b/products/endpoints/backend/metrics.py@@ -37,6 +37,15 @@ ENDPOINT_CONCURRENCY_REJECTED_TOTAL = Counter(labelnames=["team_id"],)+ENDPOINT_RATE_LIMITED_TOTAL = Counter(+ "posthog_endpoint_rate_limited_total",+ "Endpoint run requests refused with a 429. "+ "reason is materialized_rate or inline_rate (request-rate throttle; scope names the bucket), "+ "concurrency (too many concurrent queries) or read_budget (hourly per-team budget of data read by API queries). "+ "For concurrency and read_budget, scope equals reason",+ labelnames=["scope", "reason"],+)+ENDPOINT_CACHE_RESULT_TOTAL = Counter("posthog_endpoint_cache_result_total","Endpoint responses by query-result cache outcome (hit or miss)",diff --git a/products/endpoints/backend/presentation/throttles.py b/products/endpoints/backend/presentation/throttles.pyindex 3cda2d0d49e5..8a09534f1746 100644--- a/products/endpoints/backend/presentation/throttles.py+++ b/products/endpoints/backend/presentation/throttles.py@@ -5,7 +5,6 @@ ones. That classification is resolved through the facade; the throttle classesand their request-parsing helpers are pure HTTP concerns and live here."""-from prometheus_client import Counterfrom rest_framework.throttling import SimpleRateThrottlefrom posthog.rate_limit import (@@ -15,13 +14,7 @@ from posthog.rate_limit import (ProjectSecretApiKeyTeamRateThrottle,)-from products.endpoints.backend.facade.api import is_materialized_request--ENDPOINT_RATE_LIMITED_TOTAL = Counter(- "posthog_endpoint_rate_limited_total",- "Rate-limited endpoint requests",- labelnames=["scope"],-)+from products.endpoints.backend.facade.api import is_materialized_request, record_throttled_requestdef _parse_requested_version(request) -> int | None:@@ -74,17 +67,18 @@ class _MaterializedRateMixin(SimpleRateThrottle):materialized_scope: strdef allow_request(self, request, view):- if _is_materialized_endpoint_request(request, view):+ materialized = _is_materialized_endpoint_request(request, view)+ if materialized:self.rate = self.materialized_rateself.scope = self.materialized_scopeself.num_requests, self.duration = self.parse_rate(self.rate)allowed = super().allow_request(request, view)if not allowed:- try:- ENDPOINT_RATE_LIMITED_TOTAL.labels(scope=self.scope).inc()- except Exception:- pass+ team_id, endpoint_name, version = _get_endpoint_info_from_request(request, view)+ record_throttled_request(+ team_id, endpoint_name, version, materialized=materialized, scope=self.scope or "unknown"+ )return alloweddiff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.pyindex 7baf1dd0613e..66474c803abc 100644--- a/products/endpoints/backend/tests/test_endpoint_execution.py+++ b/products/endpoints/backend/tests/test_endpoint_execution.py@@ -3,11 +3,13 @@ from datetime import timedeltafrom posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom unittest import mock+from django.core.cache import cachefrom django.db import InterfaceError, OperationalErrorfrom django.utils import timezoneimport psycopgfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom rest_framework import statusfrom rest_framework.response import Response@@ -17,6 +19,7 @@ from posthog.hogql.constants import LimitContextfrom posthog.hogql.errors import ExposedHogQLErrorfrom posthog.clickhouse.client.connection import Workload+from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import get_query_tagsfrom posthog.errors import CHQueryErrorNoCommonTypefrom posthog.exceptions import APIQueriesBudgetExceeded, ClickHouseAtCapacity@@ -229,6 +232,47 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):mock_signal.assert_not_called()mock_capture.assert_not_called()+ @parameterized.expand(+ [+ ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency"),+ ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget"),+ ]+ )+ def test_query_refusal_records_reason_and_sends_one_event_per_window(self, _name, error, expected_reason):+ cache.clear()+ endpoint = create_endpoint_with_version(+ name=f"refused_{expected_reason}",+ team=self.team,+ query={"kind": "HogQLQuery", "query": "SELECT count() FROM events"},+ created_by=self.user,+ is_active=True,+ )+ labels = {"scope": expected_reason, "reason": expected_reason}+ before = REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels) or 0+ url = f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/"++ with (+ mock.patch("products.endpoints.backend.logic.execution.process_query_model", side_effect=error),+ mock.patch("products.endpoints.backend.logic.rate_limit_signal.posthoganalytics.capture") as mock_capture,+ ):+ for _ in range(2):+ response = self.client.post(url, {}, format="json")+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)++ self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)+ events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]+ self.assertEqual(len(events), 1)+ properties = events[0].kwargs["properties"]+ self.assertEqual(+ {k: properties[k] for k in ("team_id", "endpoint_name", "endpoint_version", "reason")},+ {+ "team_id": self.team.id,+ "endpoint_name": endpoint.name,+ "endpoint_version": 1,+ "reason": expected_reason,+ },+ )+def test_capacity_error_preserves_retry_after(self):endpoint = create_endpoint_with_version(name="at_capacity",diff --git a/products/endpoints/backend/tests/test_rate_limit.py b/products/endpoints/backend/tests/test_rate_limit.pyindex 475133cfccab..fa5db66c8a6a 100644--- a/products/endpoints/backend/tests/test_rate_limit.py+++ b/products/endpoints/backend/tests/test_rate_limit.py@@ -8,6 +8,7 @@ from django.test import SimpleTestCasefrom django.utils import timezonefrom parameterized import parameterized+from prometheus_client import REGISTRYfrom rest_framework.parsers import JSONParserfrom rest_framework.request import Requestfrom rest_framework.test import APIRequestFactory@@ -516,6 +517,41 @@ class TestEndpointThrottles(APIBaseTest):throttle.allow_request(request, view)self.assertEqual(throttle.scope, expected_scope)+ @parameterized.expand(+ [+ ("materialized", True, "materialized_endpoint_burst", "materialized_rate"),+ ("inline", False, "api_queries_burst", "inline_rate"),+ ]+ )+ def test_refusal_records_reason_and_sends_one_event_per_window(+ self, _name, materialized, expected_scope, expected_reason+ ):+ if materialized:+ _create_ready_materialized_endpoint(self.team, self.user, "refused", timezone.now())+ request = MagicMock()+ request.data = {}+ request.query_params = {}+ view = MagicMock()+ view.team_id = self.team.id+ view.kwargs = {"name": "refused"}+ labels = {"scope": expected_scope, "reason": expected_reason}+ before = REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels) or 0++ with (+ patch("posthog.rate_limit.PersonalOrProjectSecretApiKeyRateThrottle.allow_request", return_value=False),+ patch("products.endpoints.backend.logic.rate_limit_signal.posthoganalytics.capture") as mock_capture,+ ):+ self.assertFalse(EndpointBurstThrottle().allow_request(request, view))+ self.assertFalse(EndpointBurstThrottle().allow_request(request, view))++ self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)+ events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]+ self.assertEqual(len(events), 1)+ self.assertEqual(+ {k: events[0].kwargs["properties"][k] for k in ("team_id", "endpoint_name", "reason")},+ {"team_id": self.team.id, "endpoint_name": "refused", "reason": expected_reason},+ )+class TestMaterializationStateCacheTimeout(APIBaseTest):def setUp(self):