:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(queries): spread API concurrency limiter retries (#117119)
Vasco de Krijgera2eea6e66ad8
diff --git a/posthog/clickhouse/client/limit.py b/posthog/clickhouse/client/limit.pyindex b48fdb4ed725..12337d19b893 100644--- a/posthog/clickhouse/client/limit.py+++ b/posthog/clickhouse/client/limit.py@@ -1,5 +1,6 @@import timeimport uuid+import randomimport dataclassesfrom collections.abc import Callable, Iteratorfrom contextlib import contextmanager@@ -116,6 +117,7 @@ class RateLimit:allow_team_bypass: bool = Trueget_time: Callable[[], float] = lambda: time.time()sleep: Callable[[float], None] = lambda d: sleep(d)+ retry_jitter: bool = False@contextmanagerdef run(self, *args, **kwargs):@@ -206,6 +208,10 @@ class RateLimit:product=product_label,).inc()wait_sec = backoff(count)+ if self.retry_jitter:+ # Spread retries so released slots do not sit idle between synchronized polls.+ wait_sec = random.uniform(wait_sec / 2, wait_sec)+ wait_sec = min(wait_sec, max(0.0, wait_deadline - self.get_time()))self.sleep(wait_sec)wait_total += wait_sectag_queries(rate_limit_wait_ms=int(1000 * wait_total))@@ -274,6 +280,7 @@ def get_api_team_rate_limiter():# p20 duration for a query is 133ms, p25 is 164ms, p50 is 458ms, there's a 20% chance that after 134ms# the slot is free.retry=0.134,+ retry_jitter=True,# The default timeout for an API query on ClickHouse is 10s. p99 duration is 19s,# 15 seconds should be enough for some other query to finish. If the query cannot get a slot in this period,# the user should contact us about increasing the quota.diff --git a/posthog/clickhouse/client/test/test_limit.py b/posthog/clickhouse/client/test/test_limit.pyindex 645fbe49bf80..ea53842c56ca 100644--- a/posthog/clickhouse/client/test/test_limit.py+++ b/posthog/clickhouse/client/test/test_limit.py@@ -1,6 +1,7 @@import uuidimport asynciofrom collections.abc import Callable+from dataclasses import replacefrom posthog.test.base import BaseTestfrom unittest.mock import Mock, patch@@ -10,7 +11,13 @@ from django.test import SimpleTestCasefrom parameterized import parameterizedfrom posthog.clickhouse.client.execute import KillSwitchLevel-from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded, ConcurrencySlot, RateLimit+from posthog.clickhouse.client.limit import (+ ConcurrencyLimitExceeded,+ ConcurrencySlot,+ RateLimit,+ get_api_team_rate_limiter,+ get_materialized_endpoints_rate_limiter,+)from posthog.clickhouse.query_tagging import Product, QueryTags, query_tags, reset_query_tags, tag_queriesfrom posthog.constants import AvailableFeature@@ -198,19 +205,18 @@ class TestRateLimit(SimpleTestCase):assert result == 11- def test_retry_mechanism_raises_exception(self):- """- Test that the retry mechanism raises an exception if despite waiting for the retry timeout, the slot is not available.- """+ @parameterized.expand([(False, 1.0, 0.5), (True, 0.0, 0.5), (True, 0.0, 15.0), (True, 1.0, 15.0)])+ def test_retry_mechanism_raises_exception(self, jitter: bool, fraction: float, timeout: float) -> None:retry_limit = RateLimit(max_concurrency=1,applicable=lambda *args, **kwargs: True,limit_name="test_retry_mechanism_raises_exception",get_task_name=lambda *args, **kwargs: "test_retry_mechanism_raises_exception",get_task_id=lambda *args, **kwargs: f"task-{kwargs.get('task_id', 1)}",- ttl=10,+ ttl=60,retry=0.1, # 100ms initial retry- retry_timeout=0.5, # 500ms total timeout+ retry_timeout=timeout,+ retry_jitter=jitter,)time_helper = TimeHelper()@@ -221,27 +227,27 @@ class TestRateLimit(SimpleTestCase):with retry_limit.run(task_id=1):pass- # Second task should retry and eventually fail due to timeoutslot = retry_limit.use(task_id=2)assert slot is not Noneself.addCleanup(retry_limit.release, slot)- with self.assertRaises(ConcurrencyLimitExceeded):- with retry_limit.run(task_id=2):- pass-- # Verify exponential backoff- assert len(time_helper.sleep_times) == 3- for i in range(1, len(time_helper.sleep_times)):- assert time_helper.sleep_times[i] > time_helper.sleep_times[i - 1] # Each retry should wait longer- total_sleep_time = sum(time_helper.sleep_times[:-1])-- # Verify total time is within timeout- assert total_sleep_time <= 0.5 # Should not exceed retry_timeout-- def test_retry_mechanism_acquires_slot(self):- """- Test that the retry mechanism acquires a slot if it is available.- """+ with patch(+ "posthog.clickhouse.client.limit.random.uniform",+ side_effect=lambda low, high: low + fraction * (high - low),+ ):+ with self.assertRaises(ConcurrencyLimitExceeded):+ with retry_limit.run(task_id=3):+ pass++ assert len(time_helper.sleep_times) >= 3+ for previous, current in zip(time_helper.sleep_times[:-2], time_helper.sleep_times[1:-1]):+ assert previous <= current+ factor = (1 + fraction) / 2 if jitter else 1+ self.assertAlmostEqual(time_helper.sleep_times[0], 0.1 * factor)+ assert max(time_helper.sleep_times) <= 1.714 * factor+ assert timeout - 0.01 < sum(time_helper.sleep_times) <= timeout++ @parameterized.expand([(False,), (True,)])+ def test_retry_mechanism_acquires_slot(self, jitter: bool) -> None:retry_limit = RateLimit(max_concurrency=1,applicable=lambda *args, **kwargs: True,@@ -251,6 +257,7 @@ class TestRateLimit(SimpleTestCase):ttl=10,retry=0.1, # 100ms initial retryretry_timeout=0.5, # 500ms total timeout+ retry_jitter=jitter,)slot = retry_limit.use(task_id=1) # consumes the slot@@ -265,18 +272,29 @@ class TestRateLimit(SimpleTestCase):retry_limit.sleep = time_helper.sleepretry_limit.get_time = time_helper.get_time- with retry_limit.run(task_id=2):- pass+ with patch("posthog.clickhouse.client.limit.random.uniform", side_effect=lambda low, high: (low + high) / 2):+ with retry_limit.run(task_id=2):+ pass- # Verify exponential backoff+ self.assertAlmostEqual(time_helper.sleep_times[0], 0.075 if jitter else 0.1)assert len(time_helper.sleep_times) == 2for i in range(1, len(time_helper.sleep_times)):assert time_helper.sleep_times[i] > time_helper.sleep_times[i - 1] # Each retry should wait longer- total_sleep_time = sum(time_helper.sleep_times[:-1])+ total_sleep_time = sum(time_helper.sleep_times)- # Verify total time is within timeoutassert total_sleep_time <= 0.5 # Should not exceed retry_timeout+ @parameterized.expand([(get_api_team_rate_limiter, 0.1005), (get_materialized_endpoints_rate_limiter, 0.134)])+ def test_factory_retry_policy(self, factory: Callable[[], RateLimit], expected_wait: float) -> None:+ time_helper = TimeHelper()+ limiter = replace(factory(), get_time=time_helper.get_time, sleep=time_helper.sleep, allow_team_bypass=False)+ limiter.redis_client = Mock()+ limiter.redis_client.eval.side_effect = [0, 1]+ with patch("posthog.clickhouse.client.limit.random.uniform", side_effect=lambda low, high: (low + high) / 2):+ slot = limiter.use(task_id="waiting")+ self.assertIsNotNone(slot)+ self.assertAlmostEqual(sum(time_helper.sleep_times), expected_wait)+class TimeHelper:def __init__(self, on_sleep: Callable[[float], None] = lambda _: None):