Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): retry too-many-queries after a jittered delay (#121177)

Jovan Sakovica633ed4c2d94
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index 8dd4f91809c6..5fa54affd84f 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -1,4 +1,5 @@
import uuid
+import random
import typing
import asyncio
import datetime as dt
@@ -39,6 +40,7 @@ from posthog.settings.base_variables import TEST
from posthog.sync import database_sync_to_async_pool
from posthog.temporal.common.clickhouse import (
ClickHouseError,
+ ClickHouseTooManySimultaneousQueriesError,
get_client as get_clickhouse_client,
)
from posthog.temporal.common.db_errors import is_transient_db_error
@@ -200,6 +202,10 @@ DELTA_TABLE_RETENTION_HOURS = 24
DELTA_KERNEL_ERROR_MARKER = "DELTA_KERNEL_ERROR"
DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)
+# A saturated cluster refuses every model a spike reaches at once. The default backoff sends them
+# all back within seconds, so wait longer, and jitter the wait so they do not return together.
+TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)
+
# Above this many files, the per-run compaction is worth its full-table rewrite. Below it, skipping
# keeps an incremental run's cost proportional to the rows it changed rather than the table's size.
INCREMENTAL_COMPACT_FILE_THRESHOLD = 200
@@ -1362,6 +1368,11 @@ async def materialize_view_activity(inputs: MaterializeViewInputs) -> Materializ
raise ApplicationError(
str(error), type=type(error).__name__, next_retry_delay=DELTA_KERNEL_ERROR_RETRY_DELAY
) from error
+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError):
+ delay = random.uniform(*TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS)
+ raise ApplicationError(
+ str(error), type=type(error).__name__, next_retry_delay=dt.timedelta(seconds=delay)
+ ) from error
raise
except (Exception, asyncio.CancelledError):
if not published:
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index b8568079d589..18ece9ef1624 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -24,7 +24,7 @@ from posthog.hogql.resolver import ResolverFactory
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.models import Team, User
from posthog.sync import database_sync_to_async
-from posthog.temporal.common.clickhouse import ClickHouseError
+from posthog.temporal.common.clickhouse import ClickHouseError, ClickHouseTooManySimultaneousQueriesError
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.data_modeling.activities import (
CreateDataModelingJobInputs,
@@ -1792,17 +1792,29 @@ class TestMaterializeViewActivity:
await activity_environment.run(materialize_view_activity, inputs)
@pytest.mark.parametrize(
- "clickhouse_message,expects_delayed_retry",
+ "clickhouse_error,expects_delayed_retry",
[
(
- "Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel "
- "error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)",
+ ClickHouseError(
+ "Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel "
+ "error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)"
+ ),
True,
),
- ("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)", False),
+ (
+ ClickHouseTooManySimultaneousQueriesError(
+ "Code: 202. DB::Exception: Too many simultaneous queries. Maximum: 100. "
+ "(TOO_MANY_SIMULTANEOUS_QUERIES)"
+ ),
+ True,
+ ),
+ (
+ ClickHouseError("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)"),
+ False,
+ ),
],
)
- async def test_delta_kernel_read_error_retries_after_a_delay(
+ async def test_transient_clickhouse_errors_retry_after_a_delay(
self,
activity_environment,
ateam,
@@ -1810,11 +1822,11 @@ class TestMaterializeViewActivity:
ajob,
bucket_name,
adag,
- clickhouse_message,
+ clickhouse_error,
expects_delayed_retry,
):
def mock_hogql_table(*args, **kwargs):
- raise ClickHouseError(clickhouse_message)
+ raise clickhouse_error
with (
override_settings(
@@ -1837,15 +1849,15 @@ class TestMaterializeViewActivity:
await activity_environment.run(materialize_view_activity, inputs)
error = raised.value
- assert clickhouse_message in str(error)
+ assert str(clickhouse_error) in str(error)
if expects_delayed_retry:
assert isinstance(error, ApplicationError)
- assert error.type == "ClickHouseError"
+ assert error.type == type(clickhouse_error).__name__
assert not error.non_retryable
assert error.next_retry_delay is not None
assert error.next_retry_delay >= dt.timedelta(minutes=1)
else:
- assert type(error) is ClickHouseError
+ assert type(error) is type(clickhouse_error)
class _EmptyArrowClient: