Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(queries): spread API concurrency limiter retries (#117119)

Vasco de Krijgera2eea6e66ad8
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.py
index b48fdb4ed725..12337d19b893 100644
--- a/posthog/clickhouse/client/limit.py
+++ b/posthog/clickhouse/client/limit.py
@@ -1,5 +1,6 @@
import time
import uuid
+import random
import dataclasses
from collections.abc import Callable, Iterator
from contextlib import contextmanager
@@ -116,6 +117,7 @@ class RateLimit:
allow_team_bypass: bool = True
get_time: Callable[[], float] = lambda: time.time()
sleep: Callable[[float], None] = lambda d: sleep(d)
+ retry_jitter: bool = False
@contextmanager
def run(self, *args, **kwargs):
@@ -206,6 +208,10 @@ class RateLimit:
product=product_label,
).inc()
wait_sec = backoff(count)
+ if self.retry_jitter:
+ # Spread retries so released slots do not sit idle between synchronized polls.
+ wait_sec = random.uniform(wait_sec / 2, wait_sec)
+ wait_sec = min(wait_sec, max(0.0, wait_deadline - self.get_time()))
self.sleep(wait_sec)
wait_total += wait_sec
tag_queries(rate_limit_wait_ms=int(1000 * wait_total))
@@ -274,6 +280,7 @@ def get_api_team_rate_limiter():
# p20 duration for a query is 133ms, p25 is 164ms, p50 is 458ms, there's a 20% chance that after 134ms
# the slot is free.
retry=0.134,
+ retry_jitter=True,
# The default timeout for an API query on ClickHouse is 10s. p99 duration is 19s,
# 15 seconds should be enough for some other query to finish. If the query cannot get a slot in this period,
# the user should contact us about increasing the quota.
diff --git a/posthog/clickhouse/client/test/test_limit.py b/posthog/clickhouse/client/test/test_limit.py
index 645fbe49bf80..ea53842c56ca 100644
--- a/posthog/clickhouse/client/test/test_limit.py
+++ b/posthog/clickhouse/client/test/test_limit.py
@@ -1,6 +1,7 @@
import uuid
import asyncio
from collections.abc import Callable
+from dataclasses import replace
from posthog.test.base import BaseTest
from unittest.mock import Mock, patch
@@ -10,7 +11,13 @@ from django.test import SimpleTestCase
from parameterized import parameterized
from posthog.clickhouse.client.execute import KillSwitchLevel
-from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit
+from posthog.clickhouse.client.limit import (
+ ConcurrencyLimitExceeded,
+ ConcurrencySlot,
+ RateLimit,
+ get_api_team_rate_limiter,
+ get_materialized_endpoints_rate_limiter,
+)
from posthog.clickhouse.query_tagging import Product, QueryTags, query_tags, reset_query_tags, tag_queries
from posthog.constants import AvailableFeature
@@ -198,19 +205,18 @@ class TestRateLimit(SimpleTestCase):
assert result == 11
- def test_retry_mechanism_raises_exception(self):
- """
- Test that the retry mechanism raises an exception if despite waiting for the retry timeout, the slot is not available.
- """
+ @parameterized.expand([(False, 1.0, 0.5), (True, 0.0, 0.5), (True, 0.0, 15.0), (True, 1.0, 15.0)])
+ def test_retry_mechanism_raises_exception(self, jitter: bool, fraction: float, timeout: float) -> None:
retry_limit = RateLimit(
max_concurrency=1,
applicable=lambda *args, **kwargs: True,
limit_name="test_retry_mechanism_raises_exception",
get_task_name=lambda *args, **kwargs: "test_retry_mechanism_raises_exception",
get_task_id=lambda *args, **kwargs: f"task-{kwargs.get('task_id', 1)}",
- ttl=10,
+ ttl=60,
retry=0.1, # 100ms initial retry
- retry_timeout=0.5, # 500ms total timeout
+ retry_timeout=timeout,
+ retry_jitter=jitter,
)
time_helper = TimeHelper()
@@ -221,27 +227,27 @@ class TestRateLimit(SimpleTestCase):
with retry_limit.run(task_id=1):
pass
- # Second task should retry and eventually fail due to timeout
slot = retry_limit.use(task_id=2)
assert slot is not None
self.addCleanup(retry_limit.release, slot)
- with self.assertRaises(ConcurrencyLimitExceeded):
- with retry_limit.run(task_id=2):
- pass
-
- # Verify exponential backoff
- assert len(time_helper.sleep_times) == 3
- for i in range(1, len(time_helper.sleep_times)):
- assert time_helper.sleep_times[i] > time_helper.sleep_times[i - 1] # Each retry should wait longer
- total_sleep_time = sum(time_helper.sleep_times[:-1])
-
- # Verify total time is within timeout
- assert total_sleep_time <= 0.5 # Should not exceed retry_timeout
-
- def test_retry_mechanism_acquires_slot(self):
- """
- Test that the retry mechanism acquires a slot if it is available.
- """
+ with patch(
+ "posthog.clickhouse.client.limit.random.uniform",
+ side_effect=lambda low, high: low + fraction * (high - low),
+ ):
+ with self.assertRaises(ConcurrencyLimitExceeded):
+ with retry_limit.run(task_id=3):
+ pass
+
+ assert len(time_helper.sleep_times) >= 3
+ for previous, current in zip(time_helper.sleep_times[:-2], time_helper.sleep_times[1:-1]):
+ assert previous <= current
+ factor = (1 + fraction) / 2 if jitter else 1
+ self.assertAlmostEqual(time_helper.sleep_times[0], 0.1 * factor)
+ assert max(time_helper.sleep_times) <= 1.714 * factor
+ assert timeout - 0.01 < sum(time_helper.sleep_times) <= timeout
+
+ @parameterized.expand([(False,), (True,)])
+ def test_retry_mechanism_acquires_slot(self, jitter: bool) -> None:
retry_limit = RateLimit(
max_concurrency=1,
applicable=lambda *args, **kwargs: True,
@@ -251,6 +257,7 @@ class TestRateLimit(SimpleTestCase):
ttl=10,
retry=0.1, # 100ms initial retry
retry_timeout=0.5, # 500ms total timeout
+ retry_jitter=jitter,
)
slot = retry_limit.use(task_id=1) # consumes the slot
@@ -265,18 +272,29 @@ class TestRateLimit(SimpleTestCase):
retry_limit.sleep = time_helper.sleep
retry_limit.get_time = time_helper.get_time
- with retry_limit.run(task_id=2):
- pass
+ with patch("posthog.clickhouse.client.limit.random.uniform", side_effect=lambda low, high: (low + high) / 2):
+ with retry_limit.run(task_id=2):
+ pass
- # Verify exponential backoff
+ self.assertAlmostEqual(time_helper.sleep_times[0], 0.075 if jitter else 0.1)
assert len(time_helper.sleep_times) == 2
for i in range(1, len(time_helper.sleep_times)):
assert time_helper.sleep_times[i] > time_helper.sleep_times[i - 1] # Each retry should wait longer
- total_sleep_time = sum(time_helper.sleep_times[:-1])
+ total_sleep_time = sum(time_helper.sleep_times)
- # Verify total time is within timeout
assert total_sleep_time <= 0.5 # Should not exceed retry_timeout
+ @parameterized.expand([(get_api_team_rate_limiter, 0.1005), (get_materialized_endpoints_rate_limiter, 0.134)])
+ def test_factory_retry_policy(self, factory: Callable[[], RateLimit], expected_wait: float) -> None:
+ time_helper = TimeHelper()
+ limiter = replace(factory(), get_time=time_helper.get_time, sleep=time_helper.sleep, allow_team_bypass=False)
+ limiter.redis_client = Mock()
+ limiter.redis_client.eval.side_effect = [0, 1]
+ with patch("posthog.clickhouse.client.limit.random.uniform", side_effect=lambda low, high: (low + high) / 2):
+ slot = limiter.use(task_id="waiting")
+ self.assertIsNotNone(slot)
+ self.assertAlmostEqual(sum(time_helper.sleep_times), expected_wait)
+
class TimeHelper:
def __init__(self, on_sleep: Callable[[float], None] = lambda _: None):