:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): no scan fallback when a precompute build is rejected (#121014)
Juraj Majerika7c7b8e02c3c
diff --git a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.pyindex 714868a634d8..41a5e3243e68 100644--- a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py+++ b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py@@ -479,6 +479,10 @@ def split_ranges_by_ttl(return result+# ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES. The pool refused the INSERT before it ran, so+# nothing was written and a later attempt can pick the build up where it left off.+TOO_MANY_SIMULTANEOUS_QUERIES_CODE = 202+# ClickHouse error codes that should NOT be retried.# These are errors where retrying will never help - the query itself is broken,# or the user needs to change something about their request.@@ -495,7 +499,7 @@ NON_RETRYABLE_CLICKHOUSE_ERROR_CODES = {159, # TIMEOUT_EXCEEDED# Too many simultaneous queries means the cluster is overloaded.# Rather than adding to the load with retries, surface the error.- 202, # TOO_MANY_SIMULTANEOUS_QUERIES+ TOO_MANY_SIMULTANEOUS_QUERIES_CODE,# The rows/bytes-to-read cap is deterministic for a given window: the data# won't shrink between attempts, so an immediate retry re-scans the same# terabytes only to fail the same way. Fail fast so the caller can fall@@ -555,6 +559,16 @@ def is_memory_limit_error(error: Exception) -> bool:return False+def is_capacity_rejected_error(error: Exception) -> bool:+ """True if the error (or any wrapped cause) is a ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES."""+ current: BaseException | None = error+ while current is not None:+ if isinstance(current, ServerException) and current.code == TOO_MANY_SIMULTANEOUS_QUERIES_CODE:+ return True+ current = current.__cause__+ return False++class LazyComputationTable(StrEnum):"""Allowed target tables for lazy-computed results."""@@ -617,6 +631,9 @@ class LazyComputationResult:# True if any insert failed with ClickHouse MEMORY_LIMIT_EXCEEDED. Lets callers react# to an OOM (e.g. cap a high-cardinality team's future inserts) without parsing errors.memory_exceeded: bool = False+ # True if any insert was refused with ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES. Nothing was+ # written, so a caller can retry later instead of reading the uncached range directly.+ capacity_rejected: bool = False# True when the returned job_ids include recently-expired jobs served under the# executor's serve-stale grace instead of recomputing inline. The data is complete# but up to (TTL + grace) old; the caller decides whether to surface that.@@ -1135,6 +1152,7 @@ class LazyComputationExecutor:errors: list[str] = []failures = 0memory_exceeded = False+ capacity_rejected = Falsestart_time = time.monotonic()interval = self.poll_interval_secondssubscribed_ids: set[uuid.UUID] = set()@@ -1189,7 +1207,11 @@ class LazyComputationExecutor:if time.monotonic() - start_time >= self.wait_timeout_seconds:errors.append("Timeout waiting for computation jobs")result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("timeout", result)return result@@ -1247,7 +1269,11 @@ class LazyComputationExecutor:# report the miss so the caller serves live and warms in background.if not self.run_inserts and (build_ranges or pending_jobs):result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("check_miss", result)return result@@ -1265,7 +1291,11 @@ class LazyComputationExecutor:if time.monotonic() - start_time >= self.wait_timeout_seconds:errors.append("Timeout waiting for computation jobs")result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("timeout", result)return result@@ -1357,6 +1387,7 @@ class LazyComputationExecutor:except Exception as e:insert_elapsed = time.monotonic() - insert_startmemory_exceeded = memory_exceeded or is_memory_limit_error(e)+ capacity_rejected = capacity_rejected or is_capacity_rejected_error(e)new_job.status = PreaggregationJob.Status.FAILEDnew_job.error = str(e)new_job.save()@@ -1382,7 +1413,11 @@ class LazyComputationExecutor:if is_non_retryable_error(e):errors.append(str(e))result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("non_retryable_error", result)return result@@ -1390,7 +1425,11 @@ class LazyComputationExecutor:if failures > self.max_retries:errors.append(f"Max retries ({self.max_retries}) exceeded: {e}")result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("max_retries_exceeded", result)return result@@ -1399,7 +1438,11 @@ class LazyComputationExecutor:if build_ranges and failures > self.max_retries:errors.append("Max retries exceeded for computation")result = LazyComputationResult(- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded+ ready=False,+ job_ids=[],+ errors=errors,+ memory_exceeded=memory_exceeded,+ capacity_rejected=capacity_rejected,)_log_execution("max_retries_exceeded", result)return resultdiff --git a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.pyindex 1094c8673b0d..be702d58c235 100644--- a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py+++ b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py@@ -3795,6 +3795,22 @@ class TestExecuteOOMAndBudget(ClickhouseTestMixin, BaseTest):)assert result.ready is Falseassert result.memory_exceeded is True+ assert result.capacity_rejected is False++ def test_surfaces_capacity_rejected_on_too_many_simultaneous_queries(self):+ def rejected_insert(_t, _j) -> None:+ raise ServerException(message="Too many simultaneous queries", code=202)++ result = LazyComputationExecutor().execute(+ team=self.team,+ query_info=self._query_info(),+ start=datetime(2024, 1, 1, tzinfo=UTC),+ end=datetime(2024, 1, 2, tzinfo=UTC),+ run_insert=rejected_insert,+ )+ assert result.ready is False+ assert result.capacity_rejected is True+ assert result.memory_exceeded is Falsedef test_oom_is_not_retried(self):calls: list = []diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.pyindex 2fcb7ca5027a..c9355d2b2474 100644--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py@@ -30,6 +30,7 @@ from posthog.hogql.database.database import Databasefrom posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import Product, tag_queries, tags_context+from posthog.exceptions import ClickHouseAtCapacityfrom posthog.exceptions_capture import capture_exceptionfrom posthog.hogql_queries.query_runner import QueryRunnerfrom posthog.hogql_queries.utils.query_date_range import QueryDateRange@@ -322,8 +323,15 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):if result.ready:exposure_job_ids = [str(job_id) for job_id in result.job_ids]self._is_precomputed = True+ elif result.capacity_rejected:+ # ClickHouse refused the build because the pool is full. The direct scan+ # below would hold a slot for minutes and starve the next builds too, so+ # surface the capacity error and let the caller retry once the pool frees up.+ raise ClickHouseAtCapacity()else:logger.warning("exposure_lazy_computation_not_ready", experiment_id=self.experiment.id)+ except ClickHouseAtCapacity:+ raiseexcept Exception as e:# Swallowed: the direct-scan fallback below still returns results, which would# otherwise hide a broken precomputation path. Report so it isn't silent.@@ -350,13 +358,19 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):numerator_job_ids = [str(job_id) for job_id in ratio_result.numerator.job_ids]denominator_job_ids = [str(job_id) for job_id in ratio_result.denominator.job_ids]self._metric_events_precomputed = True+ elif ratio_result.capacity_rejected:+ raise ClickHouseAtCapacity()else:metric_result = self._ensure_metric_events_precomputed(builder)if metric_result.ready:metric_events_job_ids = [str(job_id) for job_id in metric_result.job_ids]self._metric_events_precomputed = True+ elif metric_result.capacity_rejected:+ raise ClickHouseAtCapacity()if not self._metric_events_precomputed:logger.warning("metric_events_lazy_computation_not_ready", experiment_id=self.experiment.id)+ except ClickHouseAtCapacity:+ raiseexcept Exception as e:capture_exception(e,diff --git a/products/experiments/backend/hogql_queries/metric_events_precompute.py b/products/experiments/backend/hogql_queries/metric_events_precompute.pyindex a39d44565080..e4b5cef00123 100644--- a/products/experiments/backend/hogql_queries/metric_events_precompute.py+++ b/products/experiments/backend/hogql_queries/metric_events_precompute.py@@ -59,6 +59,10 @@ class RatioPrecomputeResults:def ready(self) -> bool:return self.numerator.ready and self.denominator.ready+ @property+ def capacity_rejected(self) -> bool:+ return self.numerator.capacity_rejected or self.denominator.capacity_rejected+@frozenclass MetricEventsPrecompute:diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.pyindex 3cf26c0b7639..a6e3c53c6809 100644--- a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py+++ b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py@@ -3,7 +3,7 @@ from datetime import UTC, datetimefrom typing import Any, castfrom posthog.test.base import _create_event, _create_person-from unittest.mock import MagicMock, patch+from unittest.mock import patchfrom django.forms.models import model_to_dictfrom django.test import override_settings@@ -24,9 +24,11 @@ from posthog.schema import (from posthog.hogql.constants import MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY, get_default_hogql_global_settingsfrom posthog.clickhouse.client import sync_execute+from posthog.exceptions import ClickHouseAtCapacityfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import (+ LazyComputationResult,LazyComputationTable,_get_insert_settings,ensure_precomputed,@@ -916,7 +918,9 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):patch.object(ExperimentQueryRunner, failing_method, side_effect=Exception("boom")),# The non-failing path returns "not ready" so it neither errors nor hits ClickHouse,# keeping the assertion scoped to the single failure under test.- patch.object(ExperimentQueryRunner, other_method, return_value=MagicMock(ready=False)),+ patch.object(+ ExperimentQueryRunner, other_method, return_value=LazyComputationResult(ready=False, job_ids=[])+ ),patch("products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception") as mock_capture,@@ -932,3 +936,40 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):# be surfaced to error tracking rather than silently masked.mock_capture.assert_called_once()assert mock_capture.call_args.kwargs["additional_properties"]["precomputation_path"] == expected_path++ @parameterized.expand(+ [+ ("_ensure_exposures_precomputed", "_ensure_metric_events_precomputed"),+ ("_ensure_metric_events_precomputed", "_ensure_exposures_precomputed"),+ ]+ )+ def test_raises_at_capacity_instead_of_falling_back_when_build_is_rejected(self, rejected_method, other_method):+ feature_flag = self.create_feature_flag()+ experiment = self.create_experiment(+ feature_flag=feature_flag,+ start_date=datetime(2024, 1, 1),+ end_date=datetime(2024, 1, 5),+ )+ self._enable_precomputation()+ metric = ExperimentFunnelMetric(series=[EventsNode(event="purchase")])+ experiment.metrics = [metric.model_dump(mode="json")]+ experiment.save()++ with (+ patch.object(+ ExperimentQueryRunner,+ rejected_method,+ return_value=LazyComputationResult(ready=False, job_ids=[], capacity_rejected=True),+ ),+ patch.object(+ ExperimentQueryRunner, other_method, return_value=LazyComputationResult(ready=False, job_ids=[])+ ),+ patch(+ "products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception"+ ) as mock_capture,+ self.assertRaises(ClickHouseAtCapacity),+ ):+ self._run_experiment(experiment, metric)++ # A full pool is a retry signal, not a broken precompute path.+ mock_capture.assert_not_called()