:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(queries): bound retries after clickhouse capacity failures (#117040)
Vasco de Krijger7eb8cf587b5d
diff --git a/posthog/clickhouse/client/test/test_execute_async.py b/posthog/clickhouse/client/test/test_execute_async.pyindex e2f113eb7dc6..0c267e1648ac 100644--- a/posthog/clickhouse/client/test/test_execute_async.py+++ b/posthog/clickhouse/client/test/test_execute_async.py@@ -81,36 +81,87 @@ class TestQueryStatusManager(SimpleTestCase):@parameterized.expand([- ("at_capacity", ClickHouseAtCapacity(), ClickHouseAtCapacity.default_detail, "clickhouse_at_capacity"),- (- "ran_concurrently",- QueryRanConcurrently(),- QueryRanConcurrently.default_detail,- "query_ran_concurrently",- ),- ("concurrency_limit", ConcurrencyLimitExceeded(), None, "clickhouse_at_capacity"),+ (ClickHouseAtCapacity, 5, "clickhouse_at_capacity"),+ (ConcurrencyLimitExceeded, 11, "clickhouse_at_capacity"),+ (QueryRanConcurrently, 11, "query_ran_concurrently"),+ (RuntimeError, 1, None),])- def test_process_query_task_on_failure_marks_status_errored(self, _name, exc, error_message, error_code):+ def test_process_query_task_on_failure_marks_status_errored(self, error_type, attempts, error_code):from posthog.tasks.tasks import process_query_taskself.manager.store_query_status(self.query_status)- process_query_task.on_failure(- exc=exc,- task_id="celery-task-id",- args=(self.team_id, None, self.query_id),- kwargs={},- einfo=None,- )+ with (+ patch("posthog.clickhouse.client.execute_process_query", side_effect=error_type()) as execute,+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),+ ):+ task_result = process_query_task.apply(+ args=(self.team_id, None, self.query_id, build_query("SELECT 1"), {}, False), throw=False+ )++ self.assertTrue(task_result.failed())+ self.assertIsInstance(task_result.result, error_type)+ self.assertEqual(execute.call_count, attempts)result = self.manager.get_query_status()self.assertTrue(result.complete)self.assertTrue(result.error)- self.assertEqual(result.error_message, error_message)+ self.assertEqual(result.error_message, getattr(error_type, "default_detail", None))self.assertEqual(result.error_code, error_code)self.assertIsNotNone(result.end_time)+ @parameterized.expand([0, 1, 2, 3, 4])+ def test_process_query_task_recovers_from_capacity_errors(self, failures):+ from posthog.tasks.tasks import process_query_task++ error = ClickHouseAtCapacity()+ with (+ patch(+ "posthog.clickhouse.client.execute_process_query", side_effect=[error] * failures + [None]+ ) as execute,+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),+ patch.object(process_query_task, "retry", wraps=process_query_task.retry) as retry,+ ):+ result = process_query_task.apply(+ args=(self.team_id, None, self.query_id, build_query("SELECT 1"), {}, False), throw=False+ )++ self.assertTrue(result.successful())+ self.assertEqual(execute.call_count, failures + 1)+ self.assertEqual([call.kwargs["countdown"] for call in retry.call_args_list], [error.wait] * failures)++ @parameterized.expand([False, True])+ def test_process_query_task_respects_team_concurrency_limit(self, keyword_args):+ from posthog.tasks.tasks import process_query_task++ self.manager.store_query_status(self.query_status)+ expires_at = (datetime.datetime.now(datetime.UTC) + datetime.timedelta(minutes=5)).timestamp()+ get_client().zadd(+ f"celery_running_tasks:{process_query_task.name}:{self.team_id}",+ {f"occupied-{i}": expires_at for i in range(10)},+ )+ arguments = {+ "team_id": self.team_id,+ "user_id": None,+ "query_id": self.query_id,+ "query_json": build_query("SELECT 1"),+ "query_tags": {},+ "is_query_service": False,+ }+ with (+ patch("posthog.clickhouse.client.execute_process_query") as execute,+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),+ ):+ result = process_query_task.apply(+ **({"kwargs": arguments} if keyword_args else {"args": tuple(arguments.values())}), throw=False+ )++ self.assertTrue(result.failed())+ self.assertIsInstance(result.result, ConcurrencyLimitExceeded)+ execute.assert_not_called()+ self.assertTrue(self.manager.get_query_status().complete)+def test_store_clickhouse_query_progress(self):query_status = {f"{self.team_id}_{self.query_id}_1": {"progress": 1234}}self.manager._store_clickhouse_query_progress_dict(query_status)diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.pyindex e2079cd43187..9b348956c82c 100644--- a/posthog/tasks/tasks.py+++ b/posthog/tasks/tasks.py@@ -13,7 +13,7 @@ from django.db.models import Modelfrom django.utils import timezoneimport requests-from celery import shared_task+from celery import Task, shared_taskfrom clickhouse_driver.errors import UnknownPacketFromServerErrorfrom prometheus_client import Counter, Gaugefrom redis import Redis@@ -435,12 +435,12 @@ def _process_query_task_failure(@shared_task(+ bind=True,ignore_result=True,queue=CeleryQueue.ANALYTICS_QUERIES.value,acks_late=True,autoretry_for=(# Important: Only retry for things that might be okay on the next try- ClickHouseAtCapacity,ConcurrencyLimitExceeded,QueryRanConcurrently,),@@ -455,10 +455,11 @@ def _process_query_task_failure(@limit_concurrency(150, limit_name="global") # Do not go above what CH can handle (max_concurrent_queries)@limit_concurrency(10,- key=lambda *args, **kwargs: kwargs.get("team_id") or args[0],+ key=lambda *args, **kwargs: kwargs.get("team_id") or args[1],limit_name="per_team",) # Do not run too many queries at once for the same teamdef process_query_task(+ self: Task,team_id: int,user_id: Optional[int],query_id: str,@@ -482,16 +483,22 @@ def process_query_task(if is_query_service:tag_queries(chargeable=1)- execute_process_query(- team_id=team_id,- user_id=user_id,- query_id=query_id,- query_json=query_json,- limit_context=limit_context,- is_query_service=is_query_service,- analytics_props=analytics_props,- sharing_configuration_id=sharing_configuration_id,- )+ try:+ execute_process_query(+ team_id=team_id,+ user_id=user_id,+ query_id=query_id,+ query_json=query_json,+ limit_context=limit_context,+ is_query_service=is_query_service,+ analytics_props=analytics_props,+ sharing_configuration_id=sharing_configuration_id,+ )+ except ClickHouseAtCapacity as exc:+ # A remote capacity failure can arrive after other shards have done expensive work.+ if self.request.retries >= 4:+ raise+ raise self.retry(exc=exc, countdown=exc.wait)@shared_task(ignore_result=True)