:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): cap concurrent personal api key requests per team (#117351)
Jimmy Zhu7227bf1fce07
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.pyindex 12337d19b893..5f6fa19f2af6 100644--- a/posthog/clickhouse/client/limit.py+++ b/posthog/clickhouse/client/limit.py@@ -253,6 +253,7 @@ __APP_CONCURRENT_QUERY_PER_ORG: Optional[RateLimit] = None__APP_CONCURRENT_DASHBOARD_QUERIES_PER_ORG: Optional[RateLimit] = None__MATERIALIZED_ENDPOINTS_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None__EVENTS_LIST_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None+__LOGS_API_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None__LLM_ANALYTICS_CONCURRENT_QUERIES: Optional[RateLimit] = None@@ -378,6 +379,33 @@ def get_materialized_endpoints_rate_limiter():return __MATERIALIZED_ENDPOINTS_CONCURRENT_QUERY_PER_TEAM+def get_logs_api_team_rate_limiter() -> RateLimit:+ """+ Limits the number of concurrent logs API requests per team made with a personal API key.+ Every logs query shares one ClickHouse user on the logs cluster, and that user has a server-side+ concurrency cap. A script that sends a burst of requests can fill the cap and cause refused+ queries for logs alerting and for other teams' log searches.+ """+ global __LOGS_API_CONCURRENT_QUERY_PER_TEAM++ if __LOGS_API_CONCURRENT_QUERY_PER_TEAM is None:+ __LOGS_API_CONCURRENT_QUERY_PER_TEAM = RateLimit(+ max_concurrency=3,+ limit_name="logs_api_per_team",+ get_task_name=lambda *args, **kwargs: f"logs_api:query:per-team:{kwargs.get('team_id')}",+ get_task_id=lambda *args, **kwargs: kwargs.get("task_id") or generate_short_id(),+ # A slot is held for a whole request. The longest request is patterns_diff, which runs six sequential+ # queries (three per window) of up to 60 seconds each, the default max_execution_time. The ttl must+ # outlast that, or the slot expires while the request still queries ClickHouse. A worker that dies+ # without releasing its slot keeps it for the full ttl, so the ttl stays close to that bound.+ ttl=450,+ retry=0.134,+ # A waiting request holds a web worker, so a burst gets a quick 429 instead of a long queue.+ retry_timeout=2.0,+ )+ return __LOGS_API_CONCURRENT_QUERY_PER_TEAM++def get_events_list_rate_limiter():"""Limits the number of concurrent events_list queries per team.diff --git a/products/logs/backend/presentation/views/api.py b/products/logs/backend/presentation/views/api.pyindex 9b0eed792dc8..cccab9acb285 100644--- a/products/logs/backend/presentation/views/api.py+++ b/products/logs/backend/presentation/views/api.py@@ -2,17 +2,21 @@ import reimport jsonimport base64import datetime as dt+from typing import Anyfrom django.db import models+from django.http import HttpRequest, HttpResponseBasefrom django.utils import timezone+import structlogfrom drf_spectacular.types import OpenApiTypesfrom drf_spectacular.utils import PolymorphicProxySerializer, extend_schema, extend_schema_fieldfrom opentelemetry import tracefrom pydantic import ValidationError+from redis.exceptions import RedisErrorfrom rest_framework import serializers, status, viewsetsfrom rest_framework.decorators import action-from rest_framework.exceptions import ParseError+from rest_framework.exceptions import ParseError, Throttledfrom rest_framework.request import Requestfrom rest_framework.response import Responsefrom rest_framework.throttling import BaseThrottle@@ -24,7 +28,10 @@ from posthog.hogql.errors import QueryErrorfrom posthog.api.documentation import PropertyGroupOperator, _FallbackSerializerfrom posthog.api.mixins import PydanticModelMixinfrom posthog.api.property_value_metrics import PROPERTY_VALUES_DURATION+from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGEfrom posthog.api.routing import TeamAndOrgViewSetMixin+from posthog.auth import PersonalAPIKeyAuthentication+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, get_logs_api_team_rate_limiterfrom posthog.clickhouse.query_tagging import Feature, Product, tag_queriesfrom posthog.errors import ExposedCHQueryErrorfrom posthog.event_usage import get_request_analytics_properties, report_user_action@@ -84,6 +91,7 @@ __all__ = []tracer = trace.get_tracer(__name__)+logger = structlog.get_logger(__name__)LOGS_MAX_EXPORT_ROWS = 10_000MAX_ATTRIBUTE_KEYS = 100@@ -1350,6 +1358,7 @@ class _LogsValuesResponseSerializer(serializers.Serializer):class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):scope_object = "logs"serializer_class = _FallbackSerializer+ _logs_api_slot: ConcurrencySlot | None = Nonedef get_throttles(self) -> list[BaseThrottle]:# patterns_diff mines two windows per request (current + baseline), roughly doubling the@@ -1359,6 +1368,34 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):return [ClickHouseBurstRateThrottle(), ClickHouseSustainedRateThrottle()]return super().get_throttles()+ def initial(self, request: Request, *args: Any, **kwargs: Any) -> None:+ super().initial(request, *args, **kwargs)+ if not isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication):+ return+ try:+ self._logs_api_slot = get_logs_api_team_rate_limiter().use(team_id=self.team_id)+ except ConcurrencyLimitExceeded as error:+ logger.warning("logs_api_concurrency_limit_exceeded", team_id=self.team_id, detail=str(error))+ # wait sets a Retry-After header, so a polling client backs off instead of retrying the burst at once.+ raise Throttled(wait=1, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error+ except RedisError:+ # The cap protects ClickHouse capacity. A Redis outage must not also make the logs API fail.+ logger.exception("logs_api_concurrency_limiter_unavailable", team_id=self.team_id)++ def dispatch(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponseBase:+ # DRF skips finalize_response when the exception handler re-raises an error, so only a finally block+ # releases the slot on every path.+ try:+ return super().dispatch(request, *args, **kwargs)+ finally:+ if self._logs_api_slot is not None:+ try:+ get_logs_api_team_rate_limiter().release(self._logs_api_slot)+ except RedisError:+ # The slot expires after the limiter's ttl, so keep the response the query already produced.+ logger.exception("logs_api_concurrency_slot_release_failed", team_id=self.team_id)+ self._logs_api_slot = None+@staticmethoddef _normalize_filter_group(filter_group: object) -> dict:"""Normalize a flat filter array (from MCP) to the nested PropertyGroupFilter structure."""@@ -1450,7 +1487,8 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):except (KeyError, ValueError, json.JSONDecodeError):pass # Invalid cursor format, continue with original date range- requested_limit = min(query_data.get("limit", 1000), 2000)+ limit = query_data.get("limit")+ requested_limit = min(1000 if limit is None else limit, 2000)logs_query_params = {"dateRange": date_range,"severityLevels": query_data.get("severityLevels", []),diff --git a/products/logs/backend/test/test_count_api.py b/products/logs/backend/test/test_count_api.pyindex 6479ed17db6d..1f087b1fc481 100644--- a/products/logs/backend/test/test_count_api.py+++ b/products/logs/backend/test/test_count_api.py@@ -3,11 +3,17 @@ import jsonimport time_machinefrom posthog.test.base import APIBaseTest, ClickhouseTestMixin+from unittest.mock import MagicMock, patch++from django.test import override_settingsfrom parameterized import parameterized+from redis.exceptions import ConnectionError as RedisConnectionErrorfrom rest_framework import status+from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGEfrom posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.limit import get_logs_api_team_rate_limiter_FIXTURE_WINDOW = {"date_from": "2025-12-14T00:00:00Z", "date_to": "2025-12-19T00:00:00Z"}@@ -124,3 +130,68 @@ class TestCountApi(ClickhouseTestMixin, APIBaseTest):sparkline_sum = sum(bucket["count"] for bucket in sparkline_response.json())self.assertEqual(count_result["count"], sparkline_sum)++ def _post_count_with_personal_api_key(self):+ token = self.create_personal_api_key_with_scopes(["logs:read"])+ self.client.logout()+ return self.client.post(+ f"/api/projects/{self.team.id}/logs/count",+ data={"query": {"dateRange": _FIXTURE_WINDOW}},+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {token}",+ )++ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)+ def test_full_team_concurrency_limit_refuses_personal_api_key_requests_but_not_sessions(self):+ limiter = get_logs_api_team_rate_limiter()+ held_slots = [limiter.use(team_id=self.team.id) for _ in range(limiter.max_concurrency)]+ try:+ with patch.object(limiter, "retry_timeout", 0):+ session_response = self.client.post(+ f"/api/projects/{self.team.id}/logs/count",+ data={"query": {"dateRange": _FIXTURE_WINDOW}},+ content_type="application/json",+ )+ api_key_response = self._post_count_with_personal_api_key()+ finally:+ for slot in held_slots:+ if slot is not None:+ limiter.release(slot)++ self.assertEqual(session_response.status_code, status.HTTP_200_OK)+ self.assertEqual(api_key_response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)+ self.assertTrue(api_key_response.json()["detail"].startswith(CONCURRENCY_LIMIT_USER_MESSAGE))+ self.assertEqual(api_key_response["Retry-After"], "1")++ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)+ def test_personal_api_key_requests_release_their_concurrency_slot(self):+ limiter = get_logs_api_team_rate_limiter()++ statuses = [self._post_count_with_personal_api_key().status_code for _ in range(limiter.max_concurrency + 1)]++ self.assertEqual(statuses, [status.HTTP_200_OK] * (limiter.max_concurrency + 1))++ @override_settings(DEBUG=True)+ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)+ def test_personal_api_key_requests_release_their_concurrency_slot_when_the_handler_raises(self):+ limiter = get_logs_api_team_rate_limiter()+ self.client.raise_request_exception = False++ with patch(+ "products.logs.backend.presentation.views.api.CountQueryRunner.run", side_effect=RuntimeError("boom")+ ):+ response = self._post_count_with_personal_api_key()++ self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)++ self.assertEqual(limiter.redis_client.zcard(f"logs_api:query:per-team:{self.team.id}"), 0)++ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)+ def test_personal_api_key_requests_pass_when_the_limiter_redis_is_down(self):+ broken_redis = MagicMock()+ broken_redis.eval.side_effect = RedisConnectionError()++ with patch.object(get_logs_api_team_rate_limiter(), "redis_client", broken_redis):+ response = self._post_count_with_personal_api_key()++ self.assertEqual(response.status_code, status.HTTP_200_OK)diff --git a/products/logs/backend/test/test_logs_query_runner.py b/products/logs/backend/test/test_logs_query_runner.pyindex 21d2c3842523..f62c39a9ad27 100644--- a/products/logs/backend/test/test_logs_query_runner.py+++ b/products/logs/backend/test/test_logs_query_runner.py@@ -632,6 +632,18 @@ class TestLogsQueryRunner(ClickhouseTestMixin, APIBaseTest):self.assertTrue(response["hasMore"])self.assertEqual(len(queries), 1)+ @time_machine.travel("2025-12-16T10:33:00Z", tick=False)+ def test_logs_integration_null_limit_uses_the_default(self):+ query_params = {+ "dateRange": {"date_from": "2025-12-16 10:32:36.178572Z", "date_to": None},+ "limit": None,+ "filterGroup": {"type": "AND", "values": [{"type": "AND", "values": []}]},+ }++ response = self._make_logs_api_request(query_params)+ self.assertEqual(len(response["results"]), 51)+ self.assertFalse(response["hasMore"])+@time_machine.travel("2025-12-16T10:33:00Z", tick=False)def test_logs_slicing(self):# should slice the query, only return 100 results first time, then get the rest