Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): cap concurrent personal api key requests per team (#117351)

Jimmy Zhu7227bf1fce07
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.py
index 12337d19b893..5f6fa19f2af6 100644
--- a/posthog/clickhouse/client/limit.py
+++ b/posthog/clickhouse/client/limit.py
@@ -253,6 +253,7 @@ __APP_CONCURRENT_QUERY_PER_ORG: Optional[RateLimit] = None
__APP_CONCURRENT_DASHBOARD_QUERIES_PER_ORG: Optional[RateLimit] = None
__MATERIALIZED_ENDPOINTS_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None
__EVENTS_LIST_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None
+__LOGS_API_CONCURRENT_QUERY_PER_TEAM: Optional[RateLimit] = None
__LLM_ANALYTICS_CONCURRENT_QUERIES: Optional[RateLimit] = None
@@ -378,6 +379,33 @@ def get_materialized_endpoints_rate_limiter():
return __MATERIALIZED_ENDPOINTS_CONCURRENT_QUERY_PER_TEAM
+def get_logs_api_team_rate_limiter() -> RateLimit:
+ """
+ Limits the number of concurrent logs API requests per team made with a personal API key.
+ Every logs query shares one ClickHouse user on the logs cluster, and that user has a server-side
+ concurrency cap. A script that sends a burst of requests can fill the cap and cause refused
+ queries for logs alerting and for other teams' log searches.
+ """
+ global __LOGS_API_CONCURRENT_QUERY_PER_TEAM
+
+ if __LOGS_API_CONCURRENT_QUERY_PER_TEAM is None:
+ __LOGS_API_CONCURRENT_QUERY_PER_TEAM = RateLimit(
+ max_concurrency=3,
+ limit_name="logs_api_per_team",
+ get_task_name=lambda *args, **kwargs: f"logs_api:query:per-team:{kwargs.get('team_id')}",
+ get_task_id=lambda *args, **kwargs: kwargs.get("task_id") or generate_short_id(),
+ # A slot is held for a whole request. The longest request is patterns_diff, which runs six sequential
+ # queries (three per window) of up to 60 seconds each, the default max_execution_time. The ttl must
+ # outlast that, or the slot expires while the request still queries ClickHouse. A worker that dies
+ # without releasing its slot keeps it for the full ttl, so the ttl stays close to that bound.
+ ttl=450,
+ retry=0.134,
+ # A waiting request holds a web worker, so a burst gets a quick 429 instead of a long queue.
+ retry_timeout=2.0,
+ )
+ return __LOGS_API_CONCURRENT_QUERY_PER_TEAM
+
+
def get_events_list_rate_limiter():
"""
Limits the number of concurrent events_list queries per team.
diff --git a/products/logs/backend/presentation/views/api.py b/products/logs/backend/presentation/views/api.py
index 9b0eed792dc8..cccab9acb285 100644
--- a/products/logs/backend/presentation/views/api.py
+++ b/products/logs/backend/presentation/views/api.py
@@ -2,17 +2,21 @@ import re
import json
import base64
import datetime as dt
+from typing import Any
from django.db import models
+from django.http import HttpRequest, HttpResponseBase
from django.utils import timezone
+import structlog
from drf_spectacular.types import OpenApiTypes
from drf_spectacular.utils import PolymorphicProxySerializer, extend_schema, extend_schema_field
from opentelemetry import trace
from pydantic import ValidationError
+from redis.exceptions import RedisError
from rest_framework import serializers, status, viewsets
from rest_framework.decorators import action
-from rest_framework.exceptions import ParseError
+from rest_framework.exceptions import ParseError, Throttled
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.throttling import BaseThrottle
@@ -24,7 +28,10 @@ from posthog.hogql.errors import QueryError
from posthog.api.documentation import PropertyGroupOperator, _FallbackSerializer
from posthog.api.mixins import PydanticModelMixin
from posthog.api.property_value_metrics import PROPERTY_VALUES_DURATION
+from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.api.routing import TeamAndOrgViewSetMixin
+from posthog.auth import PersonalAPIKeyAuthentication
+from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, get_logs_api_team_rate_limiter
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.errors import ExposedCHQueryError
from posthog.event_usage import get_request_analytics_properties, report_user_action
@@ -84,6 +91,7 @@ __all__ = [
]
tracer = trace.get_tracer(__name__)
+logger = structlog.get_logger(__name__)
LOGS_MAX_EXPORT_ROWS = 10_000
MAX_ATTRIBUTE_KEYS = 100
@@ -1350,6 +1358,7 @@ class _LogsValuesResponseSerializer(serializers.Serializer):
class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):
scope_object = "logs"
serializer_class = _FallbackSerializer
+ _logs_api_slot: ConcurrencySlot | None = None
def get_throttles(self) -> list[BaseThrottle]:
# patterns_diff mines two windows per request (current + baseline), roughly doubling the
@@ -1359,6 +1368,34 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):
return [ClickHouseBurstRateThrottle(), ClickHouseSustainedRateThrottle()]
return super().get_throttles()
+ def initial(self, request: Request, *args: Any, **kwargs: Any) -> None:
+ super().initial(request, *args, **kwargs)
+ if not isinstance(request.successful_authenticator, PersonalAPIKeyAuthentication):
+ return
+ try:
+ self._logs_api_slot = get_logs_api_team_rate_limiter().use(team_id=self.team_id)
+ except ConcurrencyLimitExceeded as error:
+ logger.warning("logs_api_concurrency_limit_exceeded", team_id=self.team_id, detail=str(error))
+ # wait sets a Retry-After header, so a polling client backs off instead of retrying the burst at once.
+ raise Throttled(wait=1, detail=CONCURRENCY_LIMIT_USER_MESSAGE) from error
+ except RedisError:
+ # The cap protects ClickHouse capacity. A Redis outage must not also make the logs API fail.
+ logger.exception("logs_api_concurrency_limiter_unavailable", team_id=self.team_id)
+
+ def dispatch(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponseBase:
+ # DRF skips finalize_response when the exception handler re-raises an error, so only a finally block
+ # releases the slot on every path.
+ try:
+ return super().dispatch(request, *args, **kwargs)
+ finally:
+ if self._logs_api_slot is not None:
+ try:
+ get_logs_api_team_rate_limiter().release(self._logs_api_slot)
+ except RedisError:
+ # The slot expires after the limiter's ttl, so keep the response the query already produced.
+ logger.exception("logs_api_concurrency_slot_release_failed", team_id=self.team_id)
+ self._logs_api_slot = None
+
@staticmethod
def _normalize_filter_group(filter_group: object) -> dict:
"""Normalize a flat filter array (from MCP) to the nested PropertyGroupFilter structure."""
@@ -1450,7 +1487,8 @@ class LogsViewSet(TeamAndOrgViewSetMixin, PydanticModelMixin, viewsets.ViewSet):
except (KeyError, ValueError, json.JSONDecodeError):
pass # Invalid cursor format, continue with original date range
- requested_limit = min(query_data.get("limit", 1000), 2000)
+ limit = query_data.get("limit")
+ requested_limit = min(1000 if limit is None else limit, 2000)
logs_query_params = {
"dateRange": date_range,
"severityLevels": query_data.get("severityLevels", []),
diff --git a/products/logs/backend/test/test_count_api.py b/products/logs/backend/test/test_count_api.py
index 6479ed17db6d..1f087b1fc481 100644
--- a/products/logs/backend/test/test_count_api.py
+++ b/products/logs/backend/test/test_count_api.py
@@ -3,11 +3,17 @@ import json
import time_machine
from posthog.test.base import APIBaseTest, ClickhouseTestMixin
+from unittest.mock import MagicMock, patch
+
+from django.test import override_settings
from parameterized import parameterized
+from redis.exceptions import ConnectionError as RedisConnectionError
from rest_framework import status
+from posthog.api.query import CONCURRENCY_LIMIT_USER_MESSAGE
from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.limit import get_logs_api_team_rate_limiter
_FIXTURE_WINDOW = {"date_from": "2025-12-14T00:00:00Z", "date_to": "2025-12-19T00:00:00Z"}
@@ -124,3 +130,68 @@ class TestCountApi(ClickhouseTestMixin, APIBaseTest):
sparkline_sum = sum(bucket["count"] for bucket in sparkline_response.json())
self.assertEqual(count_result["count"], sparkline_sum)
+
+ def _post_count_with_personal_api_key(self):
+ token = self.create_personal_api_key_with_scopes(["logs:read"])
+ self.client.logout()
+ return self.client.post(
+ f"/api/projects/{self.team.id}/logs/count",
+ data={"query": {"dateRange": _FIXTURE_WINDOW}},
+ content_type="application/json",
+ HTTP_AUTHORIZATION=f"Bearer {token}",
+ )
+
+ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)
+ def test_full_team_concurrency_limit_refuses_personal_api_key_requests_but_not_sessions(self):
+ limiter = get_logs_api_team_rate_limiter()
+ held_slots = [limiter.use(team_id=self.team.id) for _ in range(limiter.max_concurrency)]
+ try:
+ with patch.object(limiter, "retry_timeout", 0):
+ session_response = self.client.post(
+ f"/api/projects/{self.team.id}/logs/count",
+ data={"query": {"dateRange": _FIXTURE_WINDOW}},
+ content_type="application/json",
+ )
+ api_key_response = self._post_count_with_personal_api_key()
+ finally:
+ for slot in held_slots:
+ if slot is not None:
+ limiter.release(slot)
+
+ self.assertEqual(session_response.status_code, status.HTTP_200_OK)
+ self.assertEqual(api_key_response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ self.assertTrue(api_key_response.json()["detail"].startswith(CONCURRENCY_LIMIT_USER_MESSAGE))
+ self.assertEqual(api_key_response["Retry-After"], "1")
+
+ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)
+ def test_personal_api_key_requests_release_their_concurrency_slot(self):
+ limiter = get_logs_api_team_rate_limiter()
+
+ statuses = [self._post_count_with_personal_api_key().status_code for _ in range(limiter.max_concurrency + 1)]
+
+ self.assertEqual(statuses, [status.HTTP_200_OK] * (limiter.max_concurrency + 1))
+
+ @override_settings(DEBUG=True)
+ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)
+ def test_personal_api_key_requests_release_their_concurrency_slot_when_the_handler_raises(self):
+ limiter = get_logs_api_team_rate_limiter()
+ self.client.raise_request_exception = False
+
+ with patch(
+ "products.logs.backend.presentation.views.api.CountQueryRunner.run", side_effect=RuntimeError("boom")
+ ):
+ response = self._post_count_with_personal_api_key()
+
+ self.assertEqual(response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+ self.assertEqual(limiter.redis_client.zcard(f"logs_api:query:per-team:{self.team.id}"), 0)
+
+ @time_machine.travel("2025-12-18T12:00:00Z", tick=False)
+ def test_personal_api_key_requests_pass_when_the_limiter_redis_is_down(self):
+ broken_redis = MagicMock()
+ broken_redis.eval.side_effect = RedisConnectionError()
+
+ with patch.object(get_logs_api_team_rate_limiter(), "redis_client", broken_redis):
+ response = self._post_count_with_personal_api_key()
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK)
diff --git a/products/logs/backend/test/test_logs_query_runner.py b/products/logs/backend/test/test_logs_query_runner.py
index 21d2c3842523..f62c39a9ad27 100644
--- a/products/logs/backend/test/test_logs_query_runner.py
+++ b/products/logs/backend/test/test_logs_query_runner.py
@@ -632,6 +632,18 @@ class TestLogsQueryRunner(ClickhouseTestMixin, APIBaseTest):
self.assertTrue(response["hasMore"])
self.assertEqual(len(queries), 1)
+ @time_machine.travel("2025-12-16T10:33:00Z", tick=False)
+ def test_logs_integration_null_limit_uses_the_default(self):
+ query_params = {
+ "dateRange": {"date_from": "2025-12-16 10:32:36.178572Z", "date_to": None},
+ "limit": None,
+ "filterGroup": {"type": "AND", "values": [{"type": "AND", "values": []}]},
+ }
+
+ response = self._make_logs_api_request(query_params)
+ self.assertEqual(len(response["results"]), 51)
+ self.assertFalse(response["hasMore"])
+
@time_machine.travel("2025-12-16T10:33:00Z", tick=False)
def test_logs_slicing(self):
# should slice the query, only return 100 results first time, then get the rest