:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-modeling): retry too-many-queries after a jittered delay (#121177)
Jovan Sakovica633ed4c2d94
diff --git a/posthog/temporal/data_modeling/activities/materialize_view.py b/posthog/temporal/data_modeling/activities/materialize_view.pyindex 8dd4f91809c6..5fa54affd84f 100644--- a/posthog/temporal/data_modeling/activities/materialize_view.py+++ b/posthog/temporal/data_modeling/activities/materialize_view.py@@ -1,4 +1,5 @@import uuid+import randomimport typingimport asyncioimport datetime as dt@@ -39,6 +40,7 @@ from posthog.settings.base_variables import TESTfrom posthog.sync import database_sync_to_async_poolfrom posthog.temporal.common.clickhouse import (ClickHouseError,+ ClickHouseTooManySimultaneousQueriesError,get_client as get_clickhouse_client,)from posthog.temporal.common.db_errors import is_transient_db_error@@ -200,6 +202,10 @@ DELTA_TABLE_RETENTION_HOURS = 24DELTA_KERNEL_ERROR_MARKER = "DELTA_KERNEL_ERROR"DELTA_KERNEL_ERROR_RETRY_DELAY = dt.timedelta(minutes=2)+# A saturated cluster refuses every model a spike reaches at once. The default backoff sends them+# all back within seconds, so wait longer, and jitter the wait so they do not return together.+TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS = (60, 120)+# Above this many files, the per-run compaction is worth its full-table rewrite. Below it, skipping# keeps an incremental run's cost proportional to the rows it changed rather than the table's size.INCREMENTAL_COMPACT_FILE_THRESHOLD = 200@@ -1362,6 +1368,11 @@ async def materialize_view_activity(inputs: MaterializeViewInputs) -> Materializraise ApplicationError(str(error), type=type(error).__name__, next_retry_delay=DELTA_KERNEL_ERROR_RETRY_DELAY) from error+ if isinstance(error, ClickHouseTooManySimultaneousQueriesError):+ delay = random.uniform(*TOO_MANY_SIMULTANEOUS_QUERIES_RETRY_DELAY_SECONDS)+ raise ApplicationError(+ str(error), type=type(error).__name__, next_retry_delay=dt.timedelta(seconds=delay)+ ) from errorraiseexcept (Exception, asyncio.CancelledError):if not published:diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.pyindex b8568079d589..18ece9ef1624 100644--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py@@ -24,7 +24,7 @@ from posthog.hogql.resolver import ResolverFactoryfrom posthog.clickhouse.query_tagging import get_query_tagsfrom posthog.models import Team, Userfrom posthog.sync import database_sync_to_async-from posthog.temporal.common.clickhouse import ClickHouseError+from posthog.temporal.common.clickhouse import ClickHouseError, ClickHouseTooManySimultaneousQueriesErrorfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.data_modeling.activities import (CreateDataModelingJobInputs,@@ -1792,17 +1792,29 @@ class TestMaterializeViewActivity:await activity_environment.run(materialize_view_activity, inputs)@pytest.mark.parametrize(- "clickhouse_message,expects_delayed_retry",+ "clickhouse_error,expects_delayed_retry",[(- "Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel "- "error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)",+ ClickHouseError(+ "Code: 742. DB::Exception: Received DeltaLake kernel error GenericError: Generic delta kernel "+ "error: No files in log segment (in snapshot). (DELTA_KERNEL_ERROR)"+ ),True,),- ("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)", False),+ (+ ClickHouseTooManySimultaneousQueriesError(+ "Code: 202. DB::Exception: Too many simultaneous queries. Maximum: 100. "+ "(TOO_MANY_SIMULTANEOUS_QUERIES)"+ ),+ True,+ ),+ (+ ClickHouseError("Code: 241. DB::Exception: Memory limit (total) exceeded. (MEMORY_LIMIT_EXCEEDED)"),+ False,+ ),],)- async def test_delta_kernel_read_error_retries_after_a_delay(+ async def test_transient_clickhouse_errors_retry_after_a_delay(self,activity_environment,ateam,@@ -1810,11 +1822,11 @@ class TestMaterializeViewActivity:ajob,bucket_name,adag,- clickhouse_message,+ clickhouse_error,expects_delayed_retry,):def mock_hogql_table(*args, **kwargs):- raise ClickHouseError(clickhouse_message)+ raise clickhouse_errorwith (override_settings(@@ -1837,15 +1849,15 @@ class TestMaterializeViewActivity:await activity_environment.run(materialize_view_activity, inputs)error = raised.value- assert clickhouse_message in str(error)+ assert str(clickhouse_error) in str(error)if expects_delayed_retry:assert isinstance(error, ApplicationError)- assert error.type == "ClickHouseError"+ assert error.type == type(clickhouse_error).__name__assert not error.non_retryableassert error.next_retry_delay is not Noneassert error.next_retry_delay >= dt.timedelta(minutes=1)else:- assert type(error) is ClickHouseError+ assert type(error) is type(clickhouse_error)class _EmptyArrowClient: