Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(queries): send retry-after on the query concurrency 429 (#122840)

Vasco de Krijgerd0cbadcdd8e6
diff --git a/posthog/api/query.py b/posthog/api/query.py
index 2dc1cc661666..b50b1f6e8fa6 100644
--- a/posthog/api/query.py
+++ b/posthog/api/query.py
@@ -106,6 +106,7 @@ tracer = trace.get_tracer(__name__)
# exception embeds an internal Redis key + task id, so we log that for debugging and surface this
# friendly message instead of leaking implementation details into the UI.
CONCURRENCY_LIMIT_USER_MESSAGE = "Too many queries are running right now — please try again in a moment."
+CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS = 5
MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_MESSAGE = (
"This managed warehouse connection is no longer available. Select a source and run the query again."
)
@@ -294,7 +295,7 @@ class QueryViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet)
def _raise_concurrency_throttled(self, exc: ConcurrencyLimitExceeded) -> NoReturn:
# Log the raw detail (Redis key + task id) for Loki, but surface a clean message to the user.
logger.warning("query_concurrency_limit_exceeded", detail=str(exc))
- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE)
+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE)
@extend_schema(
request=QueryRequest,
diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py
index d26eaaa28eaa..aefc615d419e 100644
--- a/posthog/api/test/test_query.py
+++ b/posthog/api/test/test_query.py
@@ -41,6 +41,7 @@ from posthog.schema import (
from posthog.hogql.constants import LimitContext
from posthog.api.query import (
+ CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,
CONCURRENCY_LIMIT_USER_MESSAGE,
MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_CODE,
MANAGED_WAREHOUSE_QUERY_UNAVAILABLE_MESSAGE,
@@ -150,8 +151,10 @@ class TestQuery(ClickhouseTestMixin, APIBaseTest):
{"query": HogQLQuery(query="select 1").model_dump()},
)
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ self.assertEqual(response["Retry-After"], str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS))
detail = response.json()["detail"]
- self.assertEqual(detail, CONCURRENCY_LIMIT_USER_MESSAGE)
+ self.assertTrue(detail.startswith(CONCURRENCY_LIMIT_USER_MESSAGE), detail)
+ self.assertIn(f"Expected available in {CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS} seconds.", detail)
self.assertNotIn("app:query:per-org", detail)
@parameterized.expand(
diff --git a/products/ai_observability/backend/presentation/query_concurrency.py b/products/ai_observability/backend/presentation/query_concurrency.py
index c2b34149161f..dfbd343e6ffe 100644
--- a/products/ai_observability/backend/presentation/query_concurrency.py
+++ b/products/ai_observability/backend/presentation/query_concurrency.py
@@ -6,7 +6,7 @@ from django.conf import settings
import structlog
from rest_framework.exceptions import Throttled
-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.clickhouse.client.limit import (
ConcurrencyLimitExceeded,
app_org_concurrency_slot,
@@ -42,4 +42,4 @@ def query_concurrency_slots(team: Team) -> Iterator[None]:
yield
except ConcurrencyLimitExceeded as error:
logger.warning("ai_trace_query_concurrency_limit_exceeded", detail=str(error))
- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error
+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error
diff --git a/products/ai_observability/backend/tests/test_trace_views.py b/products/ai_observability/backend/tests/test_trace_views.py
index 2c0a54cba76c..25fdcdbf179c 100644
--- a/products/ai_observability/backend/tests/test_trace_views.py
+++ b/products/ai_observability/backend/tests/test_trace_views.py
@@ -6,6 +6,7 @@ from unittest.mock import patch
from parameterized import parameterized
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit
from posthog.constants import AvailableFeature
from posthog.models import Team
@@ -179,6 +180,7 @@ class TestTraceViewSet(ClickhouseTestMixin, APIBaseTest):
response = self.client.get(self._url(_encode("trace-1")) + suffix, headers=headers)
assert response.status_code == 429
+ assert response["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)
class TestTraceNodeView(ClickhouseTestMixin, APIBaseTest):
diff --git a/products/data_catalog/backend/logic/execution.py b/products/data_catalog/backend/logic/execution.py
index 9d31a24170a2..68f2cbd19ece 100644
--- a/products/data_catalog/backend/logic/execution.py
+++ b/products/data_catalog/backend/logic/execution.py
@@ -23,6 +23,7 @@ from rest_framework.exceptions import Throttled, ValidationError
from posthog.hogql.constants import LimitContext
from posthog.hogql.errors import ExposedHogQLError
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.api.services.query import process_query_dict
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import (
@@ -119,7 +120,7 @@ def run_metric(
)
except ConcurrencyLimitExceeded:
run.record("concurrency_limited")
- raise Throttled(detail=_CONCURRENCY_LIMIT_MESSAGE)
+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=_CONCURRENCY_LIMIT_MESSAGE)
except ValidationError:
run.record("rejected")
raise
diff --git a/products/data_catalog/backend/tests/test_execution.py b/products/data_catalog/backend/tests/test_execution.py
index 6598af33df52..aeb8e98651cb 100644
--- a/products/data_catalog/backend/tests/test_execution.py
+++ b/products/data_catalog/backend/tests/test_execution.py
@@ -12,6 +12,7 @@ from rest_framework.exceptions import Throttled, ValidationError
from posthog.hogql.constants import DEFAULT_DATA_CATALOG_RETURNED_ROWS
from posthog.hogql.errors import ExposedHogQLError
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.api.services.query import process_query_dict
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
@@ -186,19 +187,20 @@ class TestMetricRunPreparation(APIBaseTest):
@parameterized.expand(
[
- ("exposed_hogql", ExposedHogQLError("no such table"), ValidationError),
- ("exposed_clickhouse", ExposedCHQueryError("memory limit"), ValidationError),
- ("concurrency", ConcurrencyLimitExceeded("too many"), Throttled),
- ("unexpected", RuntimeError("engine bug"), RuntimeError),
+ ("exposed_hogql", ExposedHogQLError("no such table"), ValidationError, None),
+ ("exposed_clickhouse", ExposedCHQueryError("memory limit"), ValidationError, None),
+ ("concurrency", ConcurrencyLimitExceeded("too many"), Throttled, CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS),
+ ("unexpected", RuntimeError("engine bug"), RuntimeError, None),
]
)
def test_engine_exceptions_keep_their_http_semantics(
- self, _name: str, engine_error: Exception, expected: type[Exception]
+ self, _name: str, engine_error: Exception, expected: type[Exception], expected_wait: int | None
) -> None:
metric = upsert_metric(team=self.team, user=self.user, name="prep", description="d", definition=_HOGQL)
with patch(_PROCESS_QUERY, side_effect=engine_error):
- with self.assertRaises(expected):
+ with self.assertRaises(expected) as raised:
run_metric(team=self.team, metric=metric, user=self.user)
+ assert getattr(raised.exception, "wait", None) == expected_wait
metric.refresh_from_db()
assert metric.last_run_at is None
diff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.py
index 36c500609f73..152c80896537 100644
--- a/products/endpoints/backend/logic/execution.py
+++ b/products/endpoints/backend/logic/execution.py
@@ -42,7 +42,7 @@ from posthog.hogql.constants import LimitContext
from posthog.hogql.errors import ExposedHogQLError, ResolutionError
from posthog.api.mixins import PydanticModelMixin
-from posthog.api.query import _process_query_request
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, _process_query_request
from posthog.api.services.query import process_query_model
from posthog.auth import SessionAuthentication
from posthog.clickhouse.client.connection import Workload
@@ -653,7 +653,10 @@ class EndpointExecutionService(PydanticModelMixin):
reason=EndpointRateLimitReason.CONCURRENCY,
scope=EndpointRateLimitReason.CONCURRENCY.value,
)
- raise Throttled(detail="Too many concurrent requests. Please try again later.")
+ raise Throttled(
+ wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,
+ detail="Too many concurrent requests. Please try again later.",
+ )
except APIQueriesBudgetExceeded:
# The platform refused this query on purpose, so it is not an endpoint fault. Leave the
# execution counter alone: the budget path counts its own refusals.
diff --git a/products/endpoints/backend/tests/test_endpoint_execution.py b/products/endpoints/backend/tests/test_endpoint_execution.py
index 68da7015754c..59abe304e28a 100644
--- a/products/endpoints/backend/tests/test_endpoint_execution.py
+++ b/products/endpoints/backend/tests/test_endpoint_execution.py
@@ -235,11 +235,13 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):
@parameterized.expand(
[
- ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency"),
- ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget"),
+ ("concurrency", ConcurrencyLimitExceeded("too many"), "concurrency", "5"),
+ ("read_budget", APIQueriesBudgetExceeded(wait=120), "read_budget", "120"),
]
)
- def test_query_refusal_records_reason_and_sends_one_event_per_window(self, _name, error, expected_reason):
+ def test_query_refusal_records_reason_and_sends_one_event_per_window(
+ self, _name, error, expected_reason, expected_retry_after
+ ):
cache.clear()
endpoint = create_endpoint_with_version(
name=f"refused_{expected_reason}",
@@ -259,6 +261,7 @@ class TestEndpointExecution(ClickhouseTestMixin, APIBaseTest):
for _ in range(2):
response = self.client.post(url, {}, format="json")
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ self.assertEqual(response["Retry-After"], expected_retry_after)
self.assertEqual(REGISTRY.get_sample_value("posthog_endpoint_rate_limited_total", labels), before + 2)
events = [c for c in mock_capture.call_args_list if c.kwargs.get("event") == "endpoint rate limited"]
diff --git a/products/marketing_analytics/backend/api.py b/products/marketing_analytics/backend/api.py
index 129d878ff70c..b0ac82a65a51 100644
--- a/products/marketing_analytics/backend/api.py
+++ b/products/marketing_analytics/backend/api.py
@@ -43,6 +43,7 @@ from posthog.hogql.query import execute_hogql_query
from posthog.api.documentation import _FallbackSerializer
from posthog.api.mixins import validated_request
from posthog.api.project import capture_team_config_diff
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.clickhouse.client.limit import (
ConcurrencyLimitExceeded,
@@ -1300,7 +1301,10 @@ class MarketingAnalyticsViewSet(TeamAndOrgViewSetMixin, GenericViewSet):
limit=data["limit"],
)
except ConcurrencyLimitExceeded as error:
- raise Throttled(detail="Too many queries are running. Try again in a moment.") from error
+ raise Throttled(
+ wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS,
+ detail="Too many queries are running. Try again in a moment.",
+ ) from error
return Response(ConversionRecordingsResponseSerializer(result).data)
@validated_request(
diff --git a/products/marketing_analytics/backend/services/test_conversion_recordings.py b/products/marketing_analytics/backend/services/test_conversion_recordings.py
index cd3c8325a312..e69e63c2547e 100644
--- a/products/marketing_analytics/backend/services/test_conversion_recordings.py
+++ b/products/marketing_analytics/backend/services/test_conversion_recordings.py
@@ -17,6 +17,7 @@ from posthog.schema import (
MarketingAnalyticsTableQuery,
)
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.clickhouse.client.execute import sync_execute, validated_client_query_id
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.preaggregation.marketing_conversions_sql import TRUNCATE_MARKETING_CONVERSIONS_TABLE_SQL
@@ -182,6 +183,7 @@ class TestConversionRecordings(ClickhouseTestMixin, APIBaseTest):
mock.return_value.run.side_effect = ConcurrencyLimitExceeded("internal limiter detail")
throttled = self.client.post(url, payload, format="json")
self.assertEqual(throttled.status_code, 429)
+ self.assertEqual(throttled["Retry-After"], str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS))
self.assertNotIn("internal limiter detail", throttled.json()["detail"])
def test_campaign_mapping_and_missing_row(self) -> None:
diff --git a/products/metrics/backend/presentation/api.py b/products/metrics/backend/presentation/api.py
index 797a824cdbcf..e45c65871d46 100644
--- a/products/metrics/backend/presentation/api.py
+++ b/products/metrics/backend/presentation/api.py
@@ -24,7 +24,7 @@ from rest_framework.response import Response
from posthog.schema import MetricsDashboardVariable, MetricsDashboardVariableSelection, MetricsDashboardVariableSource
from posthog.api.documentation import _FallbackSerializer
-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.snuffle_proxy import SNUFFLE_API_FEATURE_FLAG
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
@@ -1193,7 +1193,7 @@ class MetricsViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
date_to=data["dateTo"],
)
except ConcurrencyLimitExceeded as exc:
- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from exc
+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from exc
except ValueError as exc:
raise ParseError(str(exc))
diff --git a/products/metrics/backend/tests/test_api.py b/products/metrics/backend/tests/test_api.py
index 9fff51975487..51ada168d7d7 100644
--- a/products/metrics/backend/tests/test_api.py
+++ b/products/metrics/backend/tests/test_api.py
@@ -20,6 +20,7 @@ from posthog.schema import DateRange, MetricsDashboardVariable, MetricsDashboard
from posthog.hogql import ast
from posthog.hogql.errors import ExposedHogQLError
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.constants import AvailableFeature
from posthog.errors import CHQueryErrorTooManyBytes
@@ -131,6 +132,7 @@ class TestMetricsVariableOptionsApi(APIBaseTest):
assert cached.status_code == status.HTTP_200_OK
assert throttled.status_code == status.HTTP_429_TOO_MANY_REQUESTS
+ assert throttled["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)
assert run.call_count == 1
def test_a_failed_options_query_is_shared_with_the_next_caller(self) -> None:
diff --git a/products/product_analytics/backend/presentation/query_concurrency.py b/products/product_analytics/backend/presentation/query_concurrency.py
index 6f64b055d60f..682075eb2844 100644
--- a/products/product_analytics/backend/presentation/query_concurrency.py
+++ b/products/product_analytics/backend/presentation/query_concurrency.py
@@ -5,7 +5,7 @@ from django.conf import settings
from rest_framework.exceptions import Throttled
-from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.clickhouse.client.limit import (
ConcurrencyLimitExceeded,
app_org_concurrency_slot,
@@ -38,4 +38,4 @@ def query_concurrency_slots(team: Team) -> Iterator[None]:
):
yield
except ConcurrencyLimitExceeded as error:
- raise Throttled(detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error
+ raise Throttled(wait=CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error
diff --git a/products/product_analytics/backend/tests/test_insight_data_checks.py b/products/product_analytics/backend/tests/test_insight_data_checks.py
index e1b84bd77408..9f50fa6504ad 100644
--- a/products/product_analytics/backend/tests/test_insight_data_checks.py
+++ b/products/product_analytics/backend/tests/test_insight_data_checks.py
@@ -22,6 +22,8 @@ from django.test import SimpleTestCase
from parameterized import parameterized
from rest_framework import status
+from posthog.api.query import CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import get_query_tag_value
from posthog.constants import AvailableFeature
from posthog.errors import CHQueryErrorQueryWasCancelled, ExposedCHQueryError
@@ -1071,6 +1073,20 @@ class TestInsightDataChecks(ClickhouseTestMixin, APIBaseTest):
assert response.status_code == expected, response.content
+ def test_a_full_organization_concurrency_limit_returns_a_retry_after(self) -> None:
+ with patch(
+ "products.product_analytics.backend.presentation.query_concurrency.app_org_concurrency_slot",
+ side_effect=ConcurrencyLimitExceeded("full"),
+ ):
+ response = self.client.post(
+ f"/api/projects/{self.team.id}/insights/data_checks/",
+ {"query": self._query()},
+ format="json",
+ )
+
+ assert response.status_code == status.HTTP_429_TOO_MANY_REQUESTS, response.content
+ assert response["Retry-After"] == str(CONCURRENCY_LIMIT_RETRY_AFTER_SECONDS)
+
def test_a_member_without_query_access_can_only_check_a_saved_insight(self) -> None:
insight = Insight.objects.create(team=self.team, query=self._query())
self.organization.available_product_features = [