:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(queries): send retry-after on the query concurrency 429 (#122840)
Vasco de Krijgerd0cbadcdd8e6
diff --git a/posthog/api/query.py b/posthog/api/query.pyindex 2dc1cc661666..b50b1f6e8fa6 100644--- a/posthog/api/query.py+++ b/posthog/api/query.py@@ -106,6 +106,7 @@ tracer = trace.get_tracer(__name__)# exception embeds an internal Redis key + task id, so we log that for debugging and surface this# friendly message instead of leaking implementation details into the UI.CONCURRENCY_LIMIT_USER_MESSAGE = "Too many queries are running right now — please try again in a moment."+CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS = 5MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_MESSAGE = ("This managed warehouse connection is no longer available. Select a source and run the query again.")@@ -294,7 +295,7 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)def _raise_concurrency_throttled(self, exc: ConcurrencyLimitExceeded) -> NoReturn:# Log the raw detail (Redis key + task id) for Loki, but surface a clean message to the user.logger.warning("query_concurrency_limit_exceeded", detail=str(exc))- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE)+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE)@extend_schema(request=QueryRequest,diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.pyindex d26eaaa28eaa..aefc615d419e 100644--- a/posthog/api/test/test_query.py+++ b/posthog/api/test/test_query.py@@ -41,6 +41,7 @@ from posthog.schema import (from posthog.hogql.constants import LimitContextfrom posthog.api.query import (+ CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,CONCURRENCY_LIMIT_USER_MESSAGE,MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_CODE,MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_MESSAGE,@@ -150,8 +151,10 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):{"query": HogQLQuery(query="select 1").model_dump()},)self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)+ self.assertEqual(response["Retry-After"], str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS))detail = response.json()["detail"]- self.assertEqual(detail, CONCURRENCY_LIMIT_USER_MESSAGE)+ self.assertTrue(detail.startswith(CONCURRENCY_LIMIT_USER_MESSAGE), detail)+ self.assertIn(f"Expected available in {CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS} seconds.", detail)self.assertNotIn("app:query:per-org", detail)@parameterized.expand(diff --git a/products/ai_observability/backend/presentation/query_concurrency.py b/products/ai_observability/backend/presentation/query_concurrency.pyindex c2b34149161f..dfbd343e6ffe 100644--- a/products/ai_observability/backend/presentation/query_concurrency.py+++ b/products/ai_observability/backend/presentation/query_concurrency.py@@ -6,7 +6,7 @@ from django.conf import settingsimport structlogfrom rest_framework.exceptions import Throttled-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGEfrom posthog.clickhouse.client.limit import (ConcurrencyLimitExceeded,app_org_concurrency_slot,@@ -42,4 +42,4 @@ def query_concurrency_slots(team: Team) -> Iterator[None]:yieldexcept ConcurrencyLimitExceeded as error:logger.warning("ai_trace_query_concurrency_limit_exceeded", detail=str(error))- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from errordiff --git a/products/ai_observability/backend/tests/test_trace_views.py b/products/ai_observability/backend/tests/test_trace_views.pyindex 2c0a54cba76c..25fdcdbf179c 100644--- a/products/ai_observability/backend/tests/test_trace_views.py+++ b/products/ai_observability/backend/tests/test_trace_views.py@@ -6,6 +6,7 @@ from unittest.mock import patchfrom parameterized import parameterized+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimitfrom posthog.constants import AvailableFeaturefrom posthog.models import Team@@ -179,6 +180,7 @@ class TestTraceViewSet(ClickhouseTestMixin, APIBaseTest):response = self.client.get(self._url(_encode("trace-1")) + suffix, headers=headers)assert response.status_code == 429+ assert response["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)class TestTraceNodeView(ClickhouseTestMixin, APIBaseTest):diff --git a/products/data_catalog/backend/logic/execution.py b/products/data_catalog/backend/logic/execution.pyindex 9d31a24170a2..68f2cbd19ece 100644--- a/products/data_catalog/backend/logic/execution.py+++ b/products/data_catalog/backend/logic/execution.py@@ -23,6 +23,7 @@ from rest_framework.exceptions import Throttled, ValidationErrorfrom posthog.hogql.constants import LimitContextfrom posthog.hogql.errors import ExposedHogQLError+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.api.services.query import process_query_dictfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import (@@ -119,7 +120,7 @@ def run_metric()except ConcurrencyLimitExceeded:run.record("concurrency_limited")- raise Throttled(detail=_CONCURRENCY_LIMIT_MESSAGE)+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=_CONCURRENCY_LIMIT_MESSAGE)except ValidationError:run.record("rejected")raisediff --git a/products/data_catalog/backend/tests/test_execution.py b/products/data_catalog/backend/tests/test_execution.pyindex 6598af33df52..aeb8e98651cb 100644--- a/products/data_catalog/backend/tests/test_execution.py+++ b/products/data_catalog/backend/tests/test_execution.py@@ -12,6 +12,7 @@ from rest_framework.exceptions import Throttled, ValidationErrorfrom posthog.hogql.constants import DEFAULT_DATA_CATALOG_RETURNED_ROWSfrom posthog.hogql.errors import ExposedHogQLError+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.api.services.query import process_query_dictfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context@@ -186,19 +187,20 @@ class TestMetricRunPreparation(APIBaseTest):@parameterized.expand([- ("exposed_hogql", ExposedHogQLError("no such table"), ValidationError),- ("exposed_clickhouse", ExposedCHQueryError("memory limit"), ValidationError),- ("concurrency", ConcurrencyLimitExceeded("too many"), Throttled),- ("unexpected", RuntimeError("engine bug"), RuntimeError),+ ("exposed_hogql", ExposedHogQLError("no such table"), ValidationError, None),+ ("exposed_clickhouse", ExposedCHQueryError("memory limit"), ValidationError, None),+ ("concurrency", ConcurrencyLimitExceeded("too many"), Throttled, CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS),+ ("unexpected", RuntimeError("engine bug"), RuntimeError, None),])def test_engine_exceptions_keep_their_http_semantics(- self, _name: str, engine_error: Exception, expected: type[Exception]+ self, _name: str, engine_error: Exception, expected: type[Exception], expected_wait: int | None) -> None:metric = upsert_metric(team=self.team, user=self.user, name="prep", description="d", definition=_HOGQL)with patch(_PROCESS_QUERY, side_effect=engine_error):- with self.assertRaises(expected):+ with self.assertRaises(expected) as raised:run_metric(team=self.team, metric=metric, user=self.user)+ assert getattr(raised.exception, "wait", None) == expected_waitmetric.refresh_from_db()assert metric.last_run_at is Nonediff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.pyindex 36c500609f73..152c80896537 100644--- a/products/endpoints/backend/logic/execution.py+++ b/products/endpoints/backend/logic/execution.py@@ -42,7 +42,7 @@ from posthog.hogql.constants import LimitContextfrom posthog.hogql.errors import ExposedHogQLError, ResolutionErrorfrom posthog.api.mixins import PydanticModelMixin-from posthog.api.query import _process_query_request+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, _process_query_requestfrom posthog.api.services.query import process_query_modelfrom posthog.auth import SessionAuthenticationfrom posthog.clickhouse.client.connection import Workload@@ -653,7 +653,10 @@ class EndpointExecutionService(PydanticModelMixin):reason=EndpointRateLimitReason.CONCURRENCY,scope=EndpointRateLimitReason.CONCURRENCY.value,)- raise Throttled(detail="Too many concurrent requests. Please try again later.")+ raise Throttled(+ wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,+ detail="Too many concurrent requests. Please try again later.",+ )except APIQueriesBudgetExceeded:# The platform refused this query on purpose, so it is not an endpoint fault. Leave the# execution counter alone: the budget path counts its own refusals.diff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.pyindex 68da7015754c..59abe304e28a 100644--- a/products/endpoints/backend/tests/test_endpoint_execution.py+++ b/products/endpoints/backend/tests/test_endpoint_execution.py@@ -235,11 +235,13 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):@parameterized.expand([- ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency"),- ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget"),+ ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency", "5"),+ ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget", "120"),])- def test_query_refusal_records_reason_and_sends_one_event_per_window(self, _name, error, expected_reason):+ def test_query_refusal_records_reason_and_sends_one_event_per_window(+ self, _name, error, expected_reason, expected_retry_after+ ):cache.clear()endpoint = create_endpoint_with_version(name=f"refused_{expected_reason}",@@ -259,6 +261,7 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):for _ in range(2):response = self.client.post(url, {}, format="json")self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)+ self.assertEqual(response["Retry-After"], expected_retry_after)self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]diff --git a/products/marketing_analytics/backend/api.py b/products/marketing_analytics/backend/api.pyindex 129d878ff70c..b0ac82a65a51 100644--- a/products/marketing_analytics/backend/api.py+++ b/products/marketing_analytics/backend/api.py@@ -43,6 +43,7 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.api.documentation import _FallbackSerializerfrom posthog.api.mixins import validated_requestfrom posthog.api.project import capture_team_config_diff+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.clickhouse.client.limit import (ConcurrencyLimitExceeded,@@ -1300,7 +1301,10 @@ class MarketingAnalyticsViewSet(TeamAndOrgViewSetMixin, GenericViewSet):limit=data["limit"],)except ConcurrencyLimitExceeded as error:- raise Throttled(detail="Too many queries are running. Try again in a moment.") from error+ raise Throttled(+ wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,+ detail="Too many queries are running. Try again in a moment.",+ ) from errorreturn Response(ConversionRecordingsResponseSerializer(result).data)@validated_request(diff --git a/products/marketing_analytics/backend/services/test_conversion_recordings.py b/products/marketing_analytics/backend/services/test_conversion_recordings.pyindex cd3c8325a312..e69e63c2547e 100644--- a/products/marketing_analytics/backend/services/test_conversion_recordings.py+++ b/products/marketing_analytics/backend/services/test_conversion_recordings.py@@ -17,6 +17,7 @@ from posthog.schema import (MarketingAnalyticsTableQuery,)+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.clickhouse.client.execute import sync_execute, validated_client_query_idfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.preaggregation.marketing_conversions_sql import TRUNCATE_MARKETING_CONVERSIONS_TABLE_SQL@@ -182,6 +183,7 @@ class TestConversionRecordings(ClickhouseTestMixin, APIBaseTest):mock.return_value.run.side_effect = ConcurrencyLimitExceeded("internal limiter detail")throttled = self.client.post(url, payload, format="json")self.assertEqual(throttled.status_code, 429)+ self.assertEqual(throttled["Retry-After"], str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS))self.assertNotIn("internal limiter detail", throttled.json()["detail"])def test_campaign_mapping_and_missing_row(self) -> None:diff --git a/products/metrics/backend/presentation/api.py b/products/metrics/backend/presentation/api.pyindex 797a824cdbcf..e45c65871d46 100644--- a/products/metrics/backend/presentation/api.py+++ b/products/metrics/backend/presentation/api.py@@ -24,7 +24,7 @@ from rest_framework.response import Responsefrom posthog.schema import MetricsDashboardVariable, MetricsDashboardVariableSelection, MetricsDashboardVariableSourcefrom posthog.api.documentation import _FallbackSerializer-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGEfrom posthog.api.routing import TeamAndOrgViewSetMixinfrom posthog.api.snuffle_proxy import SNUFFLE_API_FEATURE_FLAGfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded@@ -1193,7 +1193,7 @@ class MetricsViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):date_to=data["dateTo"],)except ConcurrencyLimitExceeded as exc:- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from exc+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from excexcept ValueError as exc:raise ParseError(str(exc))diff --git a/products/metrics/backend/tests/test_api.py b/products/metrics/backend/tests/test_api.pyindex 9fff51975487..51ada168d7d7 100644--- a/products/metrics/backend/tests/test_api.py+++ b/products/metrics/backend/tests/test_api.py@@ -20,6 +20,7 @@ from posthog.schema import DateRange, MetricsDashboardVariable, MetricsDashboardfrom posthog.hogql import astfrom posthog.hogql.errors import ExposedHogQLError+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDSfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.constants import AvailableFeaturefrom posthog.errors import CHQueryErrorTooManyBytes@@ -131,6 +132,7 @@ class TestMetricsVariableOptionsApi(APIBaseTest):assert cached.status_code == status.HTTP_200_OKassert throttled.status_code == status.HTTP_429_TOO_MANY_REQUESTS+ assert throttled["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)assert run.call_count == 1def test_a_failed_options_query_is_shared_with_the_next_caller(self) -> None:diff --git a/products/product_analytics/backend/presentation/query_concurrency.py b/products/product_analytics/backend/presentation/query_concurrency.pyindex 6f64b055d60f..682075eb2844 100644--- a/products/product_analytics/backend/presentation/query_concurrency.py+++ b/products/product_analytics/backend/presentation/query_concurrency.py@@ -5,7 +5,7 @@ from django.conf import settingsfrom rest_framework.exceptions import Throttled-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGEfrom posthog.clickhouse.client.limit import (ConcurrencyLimitExceeded,app_org_concurrency_slot,@@ -38,4 +38,4 @@ def query_concurrency_slots(team: Team) -> Iterator[None]:):yieldexcept ConcurrencyLimitExceeded as error:- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from errordiff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.pyindex e1b84bd77408..9f50fa6504ad 100644--- a/products/product_analytics/backend/tests/test_insight_data_checks.py+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py@@ -22,6 +22,8 @@ from django.test import SimpleTestCasefrom parameterized import parameterizedfrom rest_framework import status+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS+from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import get_query_tag_valuefrom posthog.constants import AvailableFeaturefrom posthog.errors import CHQueryErrorQueryWasCancelled, ExposedCHQueryError@@ -1071,6 +1073,20 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):assert response.status_code == expected, response.content+ def test_a_full_organization_concurrency_limit_returns_a_retry_after(self) -> None:+ with patch(+ "products.product_analytics.backend.presentation.query_concurrency.app_org_concurrency_slot",+ side_effect=ConcurrencyLimitExceeded("full"),+ ):+ response = self.client.post(+ f"/api/projects/{self.team.id}/insights/data_checks/",+ {"query": self._query()},+ format="json",+ )++ assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS, response.content+ assert response["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)+def test_a_member_without_query_access_can_only_check_a_saved_insight(self) -> None:insight = Insight.objects.create(team=self.team, query=self._query())self.organization.available_product_features = [