Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-modeling): grow refused and timed-out query retry delays with load (#123985)

Sinan Kucukkoselerf8351fabc56a
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.py
index 83b88b4c19a6..b57efea65630 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view.py
@@ -1,8 +1,10 @@
+import time
import uuid
import random
import typing
import asyncio
import datetime as dt
+import collections
import dataclasses
from django.conf import settings
@@ -41,6 +43,7 @@ from posthog.settings.base_variables import TEST
from posthog.sync import database_sync_to_async_pool
from posthog.temporal.common.clickhouse import (
ClickHouseError,
+ ClickHouseQueryTimeoutError,
ClickHouseTooManySimultaneousQueriesError,
get_client as get_clickhouse_client,
)
@@ -203,9 +206,43 @@ DELTA_TABLE_RETENTION_HOURS = 24
DELTA_KERNEL_ERROR_MARKER = "DELTA_KERNEL_ERROR"
DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)
-# A saturated cluster refuses every model a spike reaches at once. The default backoff sends them
-# all back within seconds, so wait longer, and jitter the wait so they do not return together.
-TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)
+# A saturated cluster refuses every model a spike reaches at once, so a refused query waits minutes,
+# with jitter so the refused models do not return together. Most timeouts hit one query on its own
+# and pass on a quick retry, so a timeout starts from seconds. Both delays grow with the number of
+# refusals and timeouts this worker saw recently, so during a spike the retries stop landing in it.
+TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_BASE_DELAY = dt.timedelta(minutes=2)
+QUERY_TIMEOUT_RETRY_BASE_DELAY = dt.timedelta(seconds=20)
+RETRY_PRESSURE_WINDOW = dt.timedelta(minutes=5)
+RETRY_PRESSURE_ERRORS_PER_STEP = 5
+RETRY_PRESSURE_MAX = 12
+MAX_ADAPTIVE_RETRY_DELAY = dt.timedelta(minutes=10)
+
+
+class _RecentCapacityErrors:
+ def __init__(self, window: dt.timedelta) -> None:
+ self._window_seconds = window.total_seconds()
+ self._seen_at: collections.deque[float] = collections.deque()
+
+ def record(self, now: float) -> int:
+ """Record one error and return how many came before it inside the window."""
+ while self._seen_at and self._seen_at[0] <= now - self._window_seconds:
+ self._seen_at.popleft()
+ earlier = len(self._seen_at)
+ self._seen_at.append(now)
+ return earlier
+
+ def clear(self) -> None:
+ self._seen_at.clear()
+
+
+_recent_capacity_errors = _RecentCapacityErrors(RETRY_PRESSURE_WINDOW)
+
+
+def _adaptive_retry_delay(base: dt.timedelta, attempt: int, recent_errors: int) -> dt.timedelta:
+ pressure = min(1 + recent_errors / RETRY_PRESSURE_ERRORS_PER_STEP, RETRY_PRESSURE_MAX)
+ delay = min(base * (2 ** (max(attempt, 1) - 1)) * pressure, MAX_ADAPTIVE_RETRY_DELAY)
+ return delay * random.uniform(0.5, 1.0)
+
KILL_SWITCH_MAX_THREADS: dict[KillSwitchLevel, int] = {KillSwitchLevel.LIGHT: 16, KillSwitchLevel.FULL: 8}
@@ -1366,10 +1403,17 @@ async def materialize_view_activity(inputs: MaterializeViewInputs) -> Materializ
raise ApplicationError(
str(error), type=type(error).__name__, next_retry_delay=DELTA_KERNEL_ERROR_RETRY_DELAY
) from error
- if isinstance(error, ClickHouseTooManySimultaneousQueriesError):
- delay = random.uniform(*TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS)
+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError | ClickHouseQueryTimeoutError):
+ base_delay = (
+ TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_BASE_DELAY
+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError)
+ else QUERY_TIMEOUT_RETRY_BASE_DELAY
+ )
+ recent_errors = _recent_capacity_errors.record(time.monotonic())
raise ApplicationError(
- str(error), type=type(error).__name__, next_retry_delay=dt.timedelta(seconds=delay)
+ str(error),
+ type=type(error).__name__,
+ next_retry_delay=_adaptive_retry_delay(base_delay, activity.info().attempt, recent_errors),
) from error
raise
except (Exception, asyncio.CancelledError):
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index deb4e7738dd3..99f5e0e07aa2 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -30,6 +30,7 @@ from posthog.sync import database_sync_to_async
from posthog.temporal.common.clickhouse import (
ClickHouseClient,
ClickHouseError,
+ ClickHouseQueryTimeoutError,
ClickHouseTooManySimultaneousQueriesError,
)
from posthog.temporal.common.errors import NonReportableError
@@ -52,7 +53,10 @@ from posthog.temporal.data_modeling.activities.materialize_view import (
DuplicateOutputColumnError,
EmptyHogQLResponseColumnsError,
InvalidNodeTypeException,
+ _adaptive_retry_delay,
_describe_columns,
+ _recent_capacity_errors,
+ _RecentCapacityErrors,
get_aws_storage_options,
get_s3_client,
hogql_table,
@@ -2008,25 +2012,42 @@ class TestMaterializeViewActivity:
await activity_environment.run(materialize_view_activity, inputs)
@pytest.mark.parametrize(
- "clickhouse_error,expects_delayed_retry",
+ "clickhouse_error,attempt,expected_delay_bounds",
[
(
ClickHouseError(
"Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel "
"error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)"
),
- True,
+ 1,
+ (dt.timedelta(minutes=2), dt.timedelta(minutes=2)),
),
(
ClickHouseTooManySimultaneousQueriesError(
"Code: 202. DB::Exception: Too many simultaneous queries. Maximum: 100. "
"(TOO_MANY_SIMULTANEOUS_QUERIES)"
),
- True,
+ 1,
+ (dt.timedelta(minutes=1), dt.timedelta(minutes=2)),
+ ),
+ (
+ ClickHouseQueryTimeoutError(
+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 600 seconds, maximum: 600. (TIMEOUT_EXCEEDED)"
+ ),
+ 1,
+ (dt.timedelta(seconds=10), dt.timedelta(seconds=20)),
+ ),
+ (
+ ClickHouseQueryTimeoutError(
+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 600 seconds, maximum: 600. (TIMEOUT_EXCEEDED)"
+ ),
+ 2,
+ (dt.timedelta(seconds=20), dt.timedelta(seconds=40)),
),
(
ClickHouseError("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)"),
- False,
+ 1,
+ None,
),
],
)
@@ -2039,11 +2060,15 @@ class TestMaterializeViewActivity:
bucket_name,
adag,
clickhouse_error,
- expects_delayed_retry,
+ attempt,
+ expected_delay_bounds,
):
def mock_hogql_table(*args, **kwargs):
raise clickhouse_error
+ activity_environment.info = replace(activity_environment.info, attempt=attempt)
+ _recent_capacity_errors.clear()
+
with (
override_settings(
BUCKET_URL=f"s3://{bucket_name}",
@@ -2066,16 +2091,44 @@ class TestMaterializeViewActivity:
error = raised.value
assert str(clickhouse_error) in str(error)
- if expects_delayed_retry:
+ if expected_delay_bounds is not None:
assert isinstance(error, ApplicationError)
assert error.type == type(clickhouse_error).__name__
assert not error.non_retryable
assert error.next_retry_delay is not None
- assert error.next_retry_delay >= dt.timedelta(minutes=1)
+ min_delay, max_delay = expected_delay_bounds
+ assert min_delay <= error.next_retry_delay <= max_delay
else:
assert type(error) is type(clickhouse_error)
+class TestAdaptiveRetryDelay:
+ @pytest.mark.parametrize(
+ "base,attempt,recent_errors,jitter,expected_delay",
+ [
+ (dt.timedelta(seconds=20), 1, 0, 0.5, dt.timedelta(seconds=10)),
+ (dt.timedelta(seconds=20), 1, 0, 1.0, dt.timedelta(seconds=20)),
+ (dt.timedelta(seconds=20), 1, 10, 0.5, dt.timedelta(seconds=30)),
+ (dt.timedelta(seconds=20), 2, 10, 1.0, dt.timedelta(seconds=120)),
+ (dt.timedelta(minutes=2), 3, 1000, 0.5, dt.timedelta(minutes=5)),
+ (dt.timedelta(minutes=2), 3, 1000, 1.0, dt.timedelta(minutes=10)),
+ ],
+ )
+ async def test_delay_grows_with_the_attempt_and_recent_errors(
+ self, base, attempt, recent_errors, jitter, expected_delay
+ ):
+ with unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view.random.uniform", return_value=jitter
+ ):
+ assert _adaptive_retry_delay(base, attempt, recent_errors) == expected_delay
+
+ async def test_recent_errors_count_only_the_window(self):
+ errors = _RecentCapacityErrors(dt.timedelta(minutes=5))
+
+ assert [errors.record(now) for now in (0.0, 10.0, 20.0)] == [0, 1, 2]
+ assert errors.record(315.0) == 1
+
+
class _EmptyArrowClient:
describe_body = b"id\tInt64\n"