Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(queries): bound retries after clickhouse capacity failures (#117040)

Vasco de Krijger7eb8cf587b5d
diff --git a/posthog/clickhouse/client/test/test_execute_async.py b/posthog/clickhouse/client/test/test_execute_async.py
index e2f113eb7dc6..0c267e1648ac 100644
--- a/posthog/clickhouse/client/test/test_execute_async.py
+++ b/posthog/clickhouse/client/test/test_execute_async.py
@@ -81,36 +81,87 @@ class TestQueryStatusManager(SimpleTestCase):
@parameterized.expand(
[
- ("at_capacity", ClickHouseAtCapacity(), ClickHouseAtCapacity.default_detail, "clickhouse_at_capacity"),
- (
- "ran_concurrently",
- QueryRanConcurrently(),
- QueryRanConcurrently.default_detail,
- "query_ran_concurrently",
- ),
- ("concurrency_limit", ConcurrencyLimitExceeded(), None, "clickhouse_at_capacity"),
+ (ClickHouseAtCapacity, 5, "clickhouse_at_capacity"),
+ (ConcurrencyLimitExceeded, 11, "clickhouse_at_capacity"),
+ (QueryRanConcurrently, 11, "query_ran_concurrently"),
+ (RuntimeError, 1, None),
]
)
- def test_process_query_task_on_failure_marks_status_errored(self, _name, exc, error_message, error_code):
+ def test_process_query_task_on_failure_marks_status_errored(self, error_type, attempts, error_code):
from posthog.tasks.tasks import process_query_task
self.manager.store_query_status(self.query_status)
- process_query_task.on_failure(
- exc=exc,
- task_id="celery-task-id",
- args=(self.team_id, None, self.query_id),
- kwargs={},
- einfo=None,
- )
+ with (
+ patch("posthog.clickhouse.client.execute_process_query", side_effect=error_type()) as execute,
+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),
+ ):
+ task_result = process_query_task.apply(
+ args=(self.team_id, None, self.query_id, build_query("SELECT 1"), {}, False), throw=False
+ )
+
+ self.assertTrue(task_result.failed())
+ self.assertIsInstance(task_result.result, error_type)
+ self.assertEqual(execute.call_count, attempts)
result = self.manager.get_query_status()
self.assertTrue(result.complete)
self.assertTrue(result.error)
- self.assertEqual(result.error_message, error_message)
+ self.assertEqual(result.error_message, getattr(error_type, "default_detail", None))
self.assertEqual(result.error_code, error_code)
self.assertIsNotNone(result.end_time)
+ @parameterized.expand([0, 1, 2, 3, 4])
+ def test_process_query_task_recovers_from_capacity_errors(self, failures):
+ from posthog.tasks.tasks import process_query_task
+
+ error = ClickHouseAtCapacity()
+ with (
+ patch(
+ "posthog.clickhouse.client.execute_process_query", side_effect=[error] * failures + [None]
+ ) as execute,
+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),
+ patch.object(process_query_task, "retry", wraps=process_query_task.retry) as retry,
+ ):
+ result = process_query_task.apply(
+ args=(self.team_id, None, self.query_id, build_query("SELECT 1"), {}, False), throw=False
+ )
+
+ self.assertTrue(result.successful())
+ self.assertEqual(execute.call_count, failures + 1)
+ self.assertEqual([call.kwargs["countdown"] for call in retry.call_args_list], [error.wait] * failures)
+
+ @parameterized.expand([False, True])
+ def test_process_query_task_respects_team_concurrency_limit(self, keyword_args):
+ from posthog.tasks.tasks import process_query_task
+
+ self.manager.store_query_status(self.query_status)
+ expires_at = (datetime.datetime.now(datetime.UTC) + datetime.timedelta(minutes=5)).timestamp()
+ get_client().zadd(
+ f"celery_running_tasks:{process_query_task.name}:{self.team_id}",
+ {f"occupied-{i}": expires_at for i in range(10)},
+ )
+ arguments = {
+ "team_id": self.team_id,
+ "user_id": None,
+ "query_id": self.query_id,
+ "query_json": build_query("SELECT 1"),
+ "query_tags": {},
+ "is_query_service": False,
+ }
+ with (
+ patch("posthog.clickhouse.client.execute_process_query") as execute,
+ patch.dict(process_query_task.app.conf.changes, task_eager_propagates=False),
+ ):
+ result = process_query_task.apply(
+ **({"kwargs": arguments} if keyword_args else {"args": tuple(arguments.values())}), throw=False
+ )
+
+ self.assertTrue(result.failed())
+ self.assertIsInstance(result.result, ConcurrencyLimitExceeded)
+ execute.assert_not_called()
+ self.assertTrue(self.manager.get_query_status().complete)
+
def test_store_clickhouse_query_progress(self):
query_status = {f"{self.team_id}_{self.query_id}_1": {"progress": 1234}}
self.manager._store_clickhouse_query_progress_dict(query_status)
diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.py
index e2079cd43187..9b348956c82c 100644
--- a/posthog/tasks/tasks.py
+++ b/posthog/tasks/tasks.py
@@ -13,7 +13,7 @@ from django.db.models import Model
from django.utils import timezone
import requests
-from celery import shared_task
+from celery import Task, shared_task
from clickhouse_driver.errors import UnknownPacketFromServerError
from prometheus_client import Counter, Gauge
from redis import Redis
@@ -435,12 +435,12 @@ def _process_query_task_failure(
@shared_task(
+ bind=True,
ignore_result=True,
queue=CeleryQueue.ANALYTICS_QUERIES.value,
acks_late=True,
autoretry_for=(
# Important: Only retry for things that might be okay on the next try
- ClickHouseAtCapacity,
ConcurrencyLimitExceeded,
QueryRanConcurrently,
),
@@ -455,10 +455,11 @@ def _process_query_task_failure(
@limit_concurrency(150, limit_name="global") # Do not go above what CH can handle (max_concurrent_queries)
@limit_concurrency(
10,
- key=lambda *args, **kwargs: kwargs.get("team_id") or args[0],
+ key=lambda *args, **kwargs: kwargs.get("team_id") or args[1],
limit_name="per_team",
) # Do not run too many queries at once for the same team
def process_query_task(
+ self: Task,
team_id: int,
user_id: Optional[int],
query_id: str,
@@ -482,16 +483,22 @@ def process_query_task(
if is_query_service:
tag_queries(chargeable=1)
- execute_process_query(
- team_id=team_id,
- user_id=user_id,
- query_id=query_id,
- query_json=query_json,
- limit_context=limit_context,
- is_query_service=is_query_service,
- analytics_props=analytics_props,
- sharing_configuration_id=sharing_configuration_id,
- )
+ try:
+ execute_process_query(
+ team_id=team_id,
+ user_id=user_id,
+ query_id=query_id,
+ query_json=query_json,
+ limit_context=limit_context,
+ is_query_service=is_query_service,
+ analytics_props=analytics_props,
+ sharing_configuration_id=sharing_configuration_id,
+ )
+ except ClickHouseAtCapacity as exc:
+ # A remote capacity failure can arrive after other shards have done expensive work.
+ if self.request.retries >= 4:
+ raise
+ raise self.retry(exc=exc, countdown=exc.wait)
@shared_task(ignore_result=True)