Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(endpoints): count 429 refusals by reason and send a per-team event (#121229)

Jovan Sakovicf208f806fd03
diff --git a/products/endpoints/backend/facade/api.py b/products/endpoints/backend/facade/api.py
index 43be2c60f9e2..c334e771ccc3 100644
--- a/products/endpoints/backend/facade/api.py
+++ b/products/endpoints/backend/facade/api.py
@@ -29,6 +29,7 @@ from datetime import datetime
from typing import TYPE_CHECKING
from products.endpoints.backend import rate_limit
+from products.endpoints.backend.logic.rate_limit_signal import EndpointRateLimitReason, record_endpoint_rate_limited
from products.endpoints.backend.models import Endpoint, EndpointVersion
from . import contracts
@@ -154,6 +155,16 @@ def is_materialized_request(team_id: int, endpoint_name: str, version: int | Non
return rate_limit.check_materialized_request(team_id, endpoint_name, version, request_data)
+def record_throttled_request(
+ team_id: int | None, endpoint_name: str | None, version: int | None, *, materialized: bool, scope: str
+) -> None:
+ """Record a run request refused by a request-rate throttle."""
+ reason = EndpointRateLimitReason.MATERIALIZED_RATE if materialized else EndpointRateLimitReason.INLINE_RATE
+ record_endpoint_rate_limited(
+ team_id=team_id, endpoint_name=endpoint_name, version=version, reason=reason, scope=scope
+ )
+
+
__all__ = [
"REWRITE_CONTRACT",
"EndpointCrudService",
@@ -168,6 +179,7 @@ __all__ = [
"is_materialization_ready",
"is_materialized_request",
"list_endpoints",
+ "record_throttled_request",
"live_materialization_conditions_source",
"materialization_fix_enabled",
"suggest_materialization_fix",
diff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.py
index f5614d29a00f..5021bf37c9be 100644
--- a/products/endpoints/backend/logic/execution.py
+++ b/products/endpoints/backend/logic/execution.py
@@ -76,6 +76,7 @@ from products.endpoints.backend.exceptions import EndpointAtCapacity, EndpointQu
from products.endpoints.backend.insight_transformers import MaterializedSeriesMismatchError
from products.endpoints.backend.logic.materialized_serving import serving_state_for_version
from products.endpoints.backend.logic.pagination import EndpointPagination
+from products.endpoints.backend.logic.rate_limit_signal import EndpointRateLimitReason, record_endpoint_rate_limited
from products.endpoints.backend.logic.strategies import EndpointQueryStrategy, strategy_for
from products.endpoints.backend.logic.validation import NATIVE_CLUSTER_UNSUPPORTED
from products.endpoints.backend.logs import build_execution_message, log_endpoint_execution
@@ -637,10 +638,24 @@ class EndpointExecutionService(PydanticModelMixin):
raise ValidationError("Query resolution failed: unable to resolve table or field references.")
except ConcurrencyLimitExceeded:
ENDPOINT_CONCURRENCY_REJECTED_TOTAL.labels(team_id=str(self.team.pk)).inc()
+ record_endpoint_rate_limited(
+ team_id=self.team.pk,
+ endpoint_name=endpoint.name,
+ version=version_obj.version,
+ reason=EndpointRateLimitReason.CONCURRENCY,
+ scope=EndpointRateLimitReason.CONCURRENCY.value,
+ )
raise Throttled(detail="Too many concurrent requests. Please try again later.")
except APIQueriesBudgetExceeded:
# The platform refused this query on purpose, so it is not an endpoint fault. Leave the
# execution counter alone: the budget path counts its own refusals.
+ record_endpoint_rate_limited(
+ team_id=self.team.pk,
+ endpoint_name=endpoint.name,
+ version=version_obj.version,
+ reason=EndpointRateLimitReason.READ_BUDGET,
+ scope=EndpointRateLimitReason.READ_BUDGET.value,
+ )
raise
except tuple(_QUERY_PERFORMANCE_ERRORS) as e:
execution_status = "query_performance"
diff --git a/products/endpoints/backend/logic/rate_limit_signal.py b/products/endpoints/backend/logic/rate_limit_signal.py
new file mode 100644
index 000000000000..69e7b7426283
--- /dev/null
+++ b/products/endpoints/backend/logic/rate_limit_signal.py
@@ -0,0 +1,67 @@
+"""Signals for endpoint run requests refused with a 429.
+
+Every refusal increments ``posthog_endpoint_rate_limited_total`` (fleet view, no team label).
+A ``endpoint rate limited`` analytics event carries the team, so refusals can be grouped
+per team in the warehouse. The event is sent at most once per (team, endpoint, reason)
+per window, so a team retrying in a tight loop does not flood the event stream.
+"""
+
+from enum import StrEnum
+
+from django.core.cache import cache
+
+import structlog
+import posthoganalytics
+
+from products.endpoints.backend.metrics import ENDPOINT_RATE_LIMITED_TOTAL
+
+logger = structlog.get_logger(__name__)
+
+RATE_LIMITED_EVENT = "endpoint rate limited"
+RATE_LIMITED_EVENT_WINDOW_SECONDS = 60
+_DEDUP_CACHE_KEY = "endpoint_rate_limited_event:{team_id}:{endpoint_name}:{reason}"
+
+
+class EndpointRateLimitReason(StrEnum):
+ MATERIALIZED_RATE = "materialized_rate"
+ INLINE_RATE = "inline_rate"
+ CONCURRENCY = "concurrency"
+ READ_BUDGET = "read_budget"
+
+
+def record_endpoint_rate_limited(
+ *,
+ team_id: int | None,
+ endpoint_name: str | None,
+ version: int | None,
+ reason: EndpointRateLimitReason,
+ scope: str,
+) -> None:
+ """Count a 429 and send the deduplicated per-team event. Never raises: this runs on the request path."""
+ try:
+ ENDPOINT_RATE_LIMITED_TOTAL.labels(scope=scope, reason=reason.value).inc()
+ except Exception:
+ logger.warning("endpoint_rate_limited_metric_failed", exc_info=True)
+
+ if team_id is None or not endpoint_name:
+ return
+
+ try:
+ dedup_key = _DEDUP_CACHE_KEY.format(team_id=team_id, endpoint_name=endpoint_name, reason=reason.value)
+ if not cache.add(dedup_key, 1, timeout=RATE_LIMITED_EVENT_WINDOW_SECONDS):
+ return
+ # The SDK only enqueues here; a background thread delivers the batch.
+ posthoganalytics.capture(
+ distinct_id=f"team_{team_id}",
+ event=RATE_LIMITED_EVENT,
+ properties={
+ "team_id": team_id,
+ "endpoint_name": endpoint_name,
+ "endpoint_version": version,
+ "reason": reason.value,
+ "scope": scope,
+ "$process_person_profile": False,
+ },
+ )
+ except Exception:
+ logger.warning("endpoint_rate_limited_event_failed", exc_info=True)
diff --git a/products/endpoints/backend/metrics.py b/products/endpoints/backend/metrics.py
index 47a2bb0dafcf..74d35cef2bfb 100644
--- a/products/endpoints/backend/metrics.py
+++ b/products/endpoints/backend/metrics.py
@@ -37,6 +37,15 @@ ENDPOINT_CONCURRENCY_REJECTED_TOTAL = Counter(
labelnames=["team_id"],
)
+ENDPOINT_RATE_LIMITED_TOTAL = Counter(
+ "posthog_endpoint_rate_limited_total",
+ "Endpoint run requests refused with a 429. "
+ "reason is materialized_rate or inline_rate (request-rate throttle; scope names the bucket), "
+ "concurrency (too many concurrent queries) or read_budget (hourly per-team budget of data read by API queries). "
+ "For concurrency and read_budget, scope equals reason",
+ labelnames=["scope", "reason"],
+)
+
ENDPOINT_CACHE_RESULT_TOTAL = Counter(
"posthog_endpoint_cache_result_total",
"Endpoint responses by query-result cache outcome (hit or miss)",
diff --git a/products/endpoints/backend/presentation/throttles.py b/products/endpoints/backend/presentation/throttles.py
index 3cda2d0d49e5..8a09534f1746 100644
--- a/products/endpoints/backend/presentation/throttles.py
+++ b/products/endpoints/backend/presentation/throttles.py
@@ -5,7 +5,6 @@ ones. That classification is resolved through the facade; the throttle classes
and their request-parsing helpers are pure HTTP concerns and live here.
"""
-from prometheus_client import Counter
from rest_framework.throttling import SimpleRateThrottle
from posthog.rate_limit import (
@@ -15,13 +14,7 @@ from posthog.rate_limit import (
ProjectSecretApiKeyTeamRateThrottle,
)
-from products.endpoints.backend.facade.api import is_materialized_request
-
-ENDPOINT_RATE_LIMITED_TOTAL = Counter(
- "posthog_endpoint_rate_limited_total",
- "Rate-limited endpoint requests",
- labelnames=["scope"],
-)
+from products.endpoints.backend.facade.api import is_materialized_request, record_throttled_request
def _parse_requested_version(request) -> int | None:
@@ -74,17 +67,18 @@ class _MaterializedRateMixin(SimpleRateThrottle):
materialized_scope: str
def allow_request(self, request, view):
- if _is_materialized_endpoint_request(request, view):
+ materialized = _is_materialized_endpoint_request(request, view)
+ if materialized:
self.rate = self.materialized_rate
self.scope = self.materialized_scope
self.num_requests, self.duration = self.parse_rate(self.rate)
allowed = super().allow_request(request, view)
if not allowed:
- try:
- ENDPOINT_RATE_LIMITED_TOTAL.labels(scope=self.scope).inc()
- except Exception:
- pass
+ team_id, endpoint_name, version = _get_endpoint_info_from_request(request, view)
+ record_throttled_request(
+ team_id, endpoint_name, version, materialized=materialized, scope=self.scope or "unknown"
+ )
return allowed
diff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.py
index 7baf1dd0613e..66474c803abc 100644
--- a/products/endpoints/backend/tests/test_endpoint_execution.py
+++ b/products/endpoints/backend/tests/test_endpoint_execution.py
@@ -3,11 +3,13 @@ from datetime import timedelta
from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest import mock
+from django.core.cache import cache
from django.db import InterfaceError, OperationalError
from django.utils import timezone
import psycopg
from parameterized import parameterized
+from prometheus_client import REGISTRY
from rest_framework import status
from rest_framework.response import Response
@@ -17,6 +19,7 @@ from posthog.hogql.constants import LimitContext
from posthog.hogql.errors import ExposedHogQLError
from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.errors import CHQueryErrorNoCommonType
from posthog.exceptions import APIQueriesBudgetExceeded, ClickHouseAtCapacity
@@ -229,6 +232,47 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):
mock_signal.assert_not_called()
mock_capture.assert_not_called()
+ @parameterized.expand(
+ [
+ ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency"),
+ ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget"),
+ ]
+ )
+ def test_query_refusal_records_reason_and_sends_one_event_per_window(self, _name, error, expected_reason):
+ cache.clear()
+ endpoint = create_endpoint_with_version(
+ name=f"refused_{expected_reason}",
+ team=self.team,
+ query={"kind": "HogQLQuery", "query": "SELECT count() FROM events"},
+ created_by=self.user,
+ is_active=True,
+ )
+ labels = {"scope": expected_reason, "reason": expected_reason}
+ before = REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels) or 0
+ url = f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/run/"
+
+ with (
+ mock.patch("products.endpoints.backend.logic.execution.process_query_model", side_effect=error),
+ mock.patch("products.endpoints.backend.logic.rate_limit_signal.posthoganalytics.capture") as mock_capture,
+ ):
+ for _ in range(2):
+ response = self.client.post(url, {}, format="json")
+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+
+ self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)
+ events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]
+ self.assertEqual(len(events), 1)
+ properties = events[0].kwargs["properties"]
+ self.assertEqual(
+ {k: properties[k] for k in ("team_id", "endpoint_name", "endpoint_version", "reason")},
+ {
+ "team_id": self.team.id,
+ "endpoint_name": endpoint.name,
+ "endpoint_version": 1,
+ "reason": expected_reason,
+ },
+ )
+
def test_capacity_error_preserves_retry_after(self):
endpoint = create_endpoint_with_version(
name="at_capacity",
diff --git a/products/endpoints/backend/tests/test_rate_limit.py b/products/endpoints/backend/tests/test_rate_limit.py
index 475133cfccab..fa5db66c8a6a 100644
--- a/products/endpoints/backend/tests/test_rate_limit.py
+++ b/products/endpoints/backend/tests/test_rate_limit.py
@@ -8,6 +8,7 @@ from django.test import SimpleTestCase
from django.utils import timezone
from parameterized import parameterized
+from prometheus_client import REGISTRY
from rest_framework.parsers import JSONParser
from rest_framework.request import Request
from rest_framework.test import APIRequestFactory
@@ -516,6 +517,41 @@ class TestEndpointThrottles(APIBaseTest):
throttle.allow_request(request, view)
self.assertEqual(throttle.scope, expected_scope)
+ @parameterized.expand(
+ [
+ ("materialized", True, "materialized_endpoint_burst", "materialized_rate"),
+ ("inline", False, "api_queries_burst", "inline_rate"),
+ ]
+ )
+ def test_refusal_records_reason_and_sends_one_event_per_window(
+ self, _name, materialized, expected_scope, expected_reason
+ ):
+ if materialized:
+ _create_ready_materialized_endpoint(self.team, self.user, "refused", timezone.now())
+ request = MagicMock()
+ request.data = {}
+ request.query_params = {}
+ view = MagicMock()
+ view.team_id = self.team.id
+ view.kwargs = {"name": "refused"}
+ labels = {"scope": expected_scope, "reason": expected_reason}
+ before = REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels) or 0
+
+ with (
+ patch("posthog.rate_limit.PersonalOrProjectSecretApiKeyRateThrottle.allow_request", return_value=False),
+ patch("products.endpoints.backend.logic.rate_limit_signal.posthoganalytics.capture") as mock_capture,
+ ):
+ self.assertFalse(EndpointBurstThrottle().allow_request(request, view))
+ self.assertFalse(EndpointBurstThrottle().allow_request(request, view))
+
+ self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)
+ events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]
+ self.assertEqual(len(events), 1)
+ self.assertEqual(
+ {k: events[0].kwargs["properties"][k] for k in ("team_id", "endpoint_name", "reason")},
+ {"team_id": self.team.id, "endpoint_name": "refused", "reason": expected_reason},
+ )
+
class TestMaterializationStateCacheTimeout(APIBaseTest):
def setUp(self):