:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): grow refused and timed-out query retry delays with load (#123985)
Sinan Kucukkoselerf8351fabc56a
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex 83b88b4c19a6..b57efea65630 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -1,8 +1,10 @@+import timeimport uuidimport randomimport typingimport asyncioimport datetime as dt+import collectionsimport dataclassesfrom django.conf import settings@@ -41,6 +43,7 @@ from posthog.settings.base_variables import TESTfrom posthog.sync import database_sync_to_async_poolfrom posthog.temporal.common.clickhouse import (ClickHouseError,+ ClickHouseQueryTimeoutError,ClickHouseTooManySimultaneousQueriesError,get_client as get_clickhouse_client,)@@ -203,9 +206,43 @@ DELTA_TABLE_RETENTION_HOURS = 24DELTA_KERNEL_ERROR_MARKER = "DELTA_KERNEL_ERROR"DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)-# A saturated cluster refuses every model a spike reaches at once. The default backoff sends them-# all back within seconds, so wait longer, and jitter the wait so they do not return together.-TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)+# A saturated cluster refuses every model a spike reaches at once, so a refused query waits minutes,+# with jitter so the refused models do not return together. Most timeouts hit one query on its own+# and pass on a quick retry, so a timeout starts from seconds. Both delays grow with the number of+# refusals and timeouts this worker saw recently, so during a spike the retries stop landing in it.+TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_BASE_DELAY = dt.timedelta(minutes=2)+QUERY_TIMEOUT_RETRY_BASE_DELAY = dt.timedelta(seconds=20)+RETRY_PRESSURE_WINDOW = dt.timedelta(minutes=5)+RETRY_PRESSURE_ERRORS_PER_STEP = 5+RETRY_PRESSURE_MAX = 12+MAX_ADAPTIVE_RETRY_DELAY = dt.timedelta(minutes=10)+++class _RecentCapacityErrors:+ def __init__(self, window: dt.timedelta) -> None:+ self._window_seconds = window.total_seconds()+ self._seen_at: collections.deque[float] = collections.deque()++ def record(self, now: float) -> int:+ """Record one error and return how many came before it inside the window."""+ while self._seen_at and self._seen_at[0] <= now - self._window_seconds:+ self._seen_at.popleft()+ earlier = len(self._seen_at)+ self._seen_at.append(now)+ return earlier++ def clear(self) -> None:+ self._seen_at.clear()+++_recent_capacity_errors = _RecentCapacityErrors(RETRY_PRESSURE_WINDOW)+++def _adaptive_retry_delay(base: dt.timedelta, attempt: int, recent_errors: int) -> dt.timedelta:+ pressure = min(1 + recent_errors / RETRY_PRESSURE_ERRORS_PER_STEP, RETRY_PRESSURE_MAX)+ delay = min(base * (2 ** (max(attempt, 1) - 1)) * pressure, MAX_ADAPTIVE_RETRY_DELAY)+ return delay * random.uniform(0.5, 1.0)+KILL_SWITCH_MAX_THREADS: dict[KillSwitchLevel, int] = {KillSwitchLevel.LIGHT: 16, KillSwitchLevel.FULL: 8}@@ -1366,10 +1403,17 @@ async def materialize_view_activity(inputs: MaterializeViewInputs) -> Materializraise ApplicationError(str(error), type=type(error).__name__, next_retry_delay=DELTA_KERNEL_ERROR_RETRY_DELAY) from error- if isinstance(error, ClickHouseTooManySimultaneousQueriesError):- delay = random.uniform(*TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS)+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError | ClickHouseQueryTimeoutError):+ base_delay = (+ TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_BASE_DELAY+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError)+ else QUERY_TIMEOUT_RETRY_BASE_DELAY+ )+ recent_errors = _recent_capacity_errors.record(time.monotonic())raise ApplicationError(- str(error), type=type(error).__name__, next_retry_delay=dt.timedelta(seconds=delay)+ str(error),+ type=type(error).__name__,+ next_retry_delay=_adaptive_retry_delay(base_delay, activity.info().attempt, recent_errors),) from errorraiseexcept (Exception, asyncio.CancelledError):diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex deb4e7738dd3..99f5e0e07aa2 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -30,6 +30,7 @@ from posthog.sync import database_sync_to_asyncfrom posthog.temporal.common.clickhouse import (ClickHouseClient,ClickHouseError,+ ClickHouseQueryTimeoutError,ClickHouseTooManySimultaneousQueriesError,)from posthog.temporal.common.errors import NonReportableError@@ -52,7 +53,10 @@ from posthog.temporal.data_modeling.activities.materialize_view import (DuplicateOutputColumnError,EmptyHogQLResponseColumnsError,InvalidNodeTypeException,+ _adaptive_retry_delay,_describe_columns,+ _recent_capacity_errors,+ _RecentCapacityErrors,get_aws_storage_options,get_s3_client,hogql_table,@@ -2008,25 +2012,42 @@ class TestMaterializeViewActivity:await activity_environment.run(materialize_view_activity, inputs)@pytest.mark.parametrize(- "clickhouse_error,expects_delayed_retry",+ "clickhouse_error,attempt,expected_delay_bounds",[(ClickHouseError("Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel ""error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)"),- True,+ 1,+ (dt.timedelta(minutes=2), dt.timedelta(minutes=2)),),(ClickHouseTooManySimultaneousQueriesError("Code: 202. DB::Exception: Too many simultaneous queries. Maximum: 100. ""(TOO_MANY_SIMULTANEOUS_QUERIES)"),- True,+ 1,+ (dt.timedelta(minutes=1), dt.timedelta(minutes=2)),+ ),+ (+ ClickHouseQueryTimeoutError(+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 600 seconds, maximum: 600. (TIMEOUT_EXCEEDED)"+ ),+ 1,+ (dt.timedelta(seconds=10), dt.timedelta(seconds=20)),+ ),+ (+ ClickHouseQueryTimeoutError(+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 600 seconds, maximum: 600. (TIMEOUT_EXCEEDED)"+ ),+ 2,+ (dt.timedelta(seconds=20), dt.timedelta(seconds=40)),),(ClickHouseError("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)"),- False,+ 1,+ None,),],)@@ -2039,11 +2060,15 @@ class TestMaterializeViewActivity:bucket_name,adag,clickhouse_error,- expects_delayed_retry,+ attempt,+ expected_delay_bounds,):def mock_hogql_table(*args, **kwargs):raise clickhouse_error+ activity_environment.info = replace(activity_environment.info, attempt=attempt)+ _recent_capacity_errors.clear()+with (override_settings(BUCKET_URL=f"s3://{bucket_name}",@@ -2066,16 +2091,44 @@ class TestMaterializeViewActivity:error = raised.valueassert str(clickhouse_error) in str(error)- if expects_delayed_retry:+ if expected_delay_bounds is not None:assert isinstance(error, ApplicationError)assert error.type == type(clickhouse_error).__name__assert not error.non_retryableassert error.next_retry_delay is not None- assert error.next_retry_delay >= dt.timedelta(minutes=1)+ min_delay, max_delay = expected_delay_bounds+ assert min_delay <= error.next_retry_delay <= max_delayelse:assert type(error) is type(clickhouse_error)+class TestAdaptiveRetryDelay:+ @pytest.mark.parametrize(+ "base,attempt,recent_errors,jitter,expected_delay",+ [+ (dt.timedelta(seconds=20), 1, 0, 0.5, dt.timedelta(seconds=10)),+ (dt.timedelta(seconds=20), 1, 0, 1.0, dt.timedelta(seconds=20)),+ (dt.timedelta(seconds=20), 1, 10, 0.5, dt.timedelta(seconds=30)),+ (dt.timedelta(seconds=20), 2, 10, 1.0, dt.timedelta(seconds=120)),+ (dt.timedelta(minutes=2), 3, 1000, 0.5, dt.timedelta(minutes=5)),+ (dt.timedelta(minutes=2), 3, 1000, 1.0, dt.timedelta(minutes=10)),+ ],+ )+ async def test_delay_grows_with_the_attempt_and_recent_errors(+ self, base, attempt, recent_errors, jitter, expected_delay+ ):+ with unittest.mock.patch(+ "posthog.temporal.data_modeling.activities.materialize_view.random.uniform", return_value=jitter+ ):+ assert _adaptive_retry_delay(base, attempt, recent_errors) == expected_delay++ async def test_recent_errors_count_only_the_window(self):+ errors = _RecentCapacityErrors(dt.timedelta(minutes=5))++ assert [errors.record(now) for now in (0.0, 10.0, 20.0)] == [0, 1, 2]+ assert errors.record(315.0) == 1++class _EmptyArrowClient:describe_body = b"id\tInt64\n"