Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): no scan fallback when a precompute build is rejected (#121014)

Juraj Majerika7c7b8e02c3c
diff --git a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
index 714868a634d8..41a5e3243e68 100644
--- a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
+++ b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
@@ -479,6 +479,10 @@ def split_ranges_by_ttl(
return result
+# ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES. The pool refused the INSERT before it ran, so
+# nothing was written and a later attempt can pick the build up where it left off.
+TOO_MANY_SIMULTANEOUS_QUERIES_CODE = 202
+
# ClickHouse error codes that should NOT be retried.
# These are errors where retrying will never help - the query itself is broken,
# or the user needs to change something about their request.
@@ -495,7 +499,7 @@ NON_RETRYABLE_CLICKHOUSE_ERROR_CODES = {
159, # TIMEOUT_EXCEEDED
# Too many simultaneous queries means the cluster is overloaded.
# Rather than adding to the load with retries, surface the error.
- 202, # TOO_MANY_SIMULTANEOUS_QUERIES
+ TOO_MANY_SIMULTANEOUS_QUERIES_CODE,
# The rows/bytes-to-read cap is deterministic for a given window: the data
# won't shrink between attempts, so an immediate retry re-scans the same
# terabytes only to fail the same way. Fail fast so the caller can fall
@@ -555,6 +559,16 @@ def is_memory_limit_error(error: Exception) -> bool:
return False
+def is_capacity_rejected_error(error: Exception) -> bool:
+ """True if the error (or any wrapped cause) is a ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES."""
+ current: BaseException | None = error
+ while current is not None:
+ if isinstance(current, ServerException) and current.code == TOO_MANY_SIMULTANEOUS_QUERIES_CODE:
+ return True
+ current = current.__cause__
+ return False
+
+
class LazyComputationTable(StrEnum):
"""Allowed target tables for lazy-computed results."""
@@ -617,6 +631,9 @@ class LazyComputationResult:
# True if any insert failed with ClickHouse MEMORY_LIMIT_EXCEEDED. Lets callers react
# to an OOM (e.g. cap a high-cardinality team's future inserts) without parsing errors.
memory_exceeded: bool = False
+ # True if any insert was refused with ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES. Nothing was
+ # written, so a caller can retry later instead of reading the uncached range directly.
+ capacity_rejected: bool = False
# True when the returned job_ids include recently-expired jobs served under the
# executor's serve-stale grace instead of recomputing inline. The data is complete
# but up to (TTL + grace) old; the caller decides whether to surface that.
@@ -1135,6 +1152,7 @@ class LazyComputationExecutor:
errors: list[str] = []
failures = 0
memory_exceeded = False
+ capacity_rejected = False
start_time = time.monotonic()
interval = self.poll_interval_seconds
subscribed_ids: set[uuid.UUID] = set()
@@ -1189,7 +1207,11 @@ class LazyComputationExecutor:
if time.monotonic() - start_time >= self.wait_timeout_seconds:
errors.append("Timeout waiting for computation jobs")
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("timeout", result)
return result
@@ -1247,7 +1269,11 @@ class LazyComputationExecutor:
# report the miss so the caller serves live and warms in background.
if not self.run_inserts and (build_ranges or pending_jobs):
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("check_miss", result)
return result
@@ -1265,7 +1291,11 @@ class LazyComputationExecutor:
if time.monotonic() - start_time >= self.wait_timeout_seconds:
errors.append("Timeout waiting for computation jobs")
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("timeout", result)
return result
@@ -1357,6 +1387,7 @@ class LazyComputationExecutor:
except Exception as e:
insert_elapsed = time.monotonic() - insert_start
memory_exceeded = memory_exceeded or is_memory_limit_error(e)
+ capacity_rejected = capacity_rejected or is_capacity_rejected_error(e)
new_job.status = PreaggregationJob.Status.FAILED
new_job.error = str(e)
new_job.save()
@@ -1382,7 +1413,11 @@ class LazyComputationExecutor:
if is_non_retryable_error(e):
errors.append(str(e))
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("non_retryable_error", result)
return result
@@ -1390,7 +1425,11 @@ class LazyComputationExecutor:
if failures > self.max_retries:
errors.append(f"Max retries ({self.max_retries}) exceeded: {e}")
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("max_retries_exceeded", result)
return result
@@ -1399,7 +1438,11 @@ class LazyComputationExecutor:
if build_ranges and failures > self.max_retries:
errors.append("Max retries exceeded for computation")
result = LazyComputationResult(
- ready=False, job_ids=[], errors=errors, memory_exceeded=memory_exceeded
+ ready=False,
+ job_ids=[],
+ errors=errors,
+ memory_exceeded=memory_exceeded,
+ capacity_rejected=capacity_rejected,
)
_log_execution("max_retries_exceeded", result)
return result
diff --git a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
index 1094c8673b0d..be702d58c235 100644
--- a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
+++ b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
@@ -3795,6 +3795,22 @@ class TestExecuteOOMAndBudget(ClickhouseTestMixin, BaseTest):
)
assert result.ready is False
assert result.memory_exceeded is True
+ assert result.capacity_rejected is False
+
+ def test_surfaces_capacity_rejected_on_too_many_simultaneous_queries(self):
+ def rejected_insert(_t, _j) -> None:
+ raise ServerException(message="Too many simultaneous queries", code=202)
+
+ result = LazyComputationExecutor().execute(
+ team=self.team,
+ query_info=self._query_info(),
+ start=datetime(2024, 1, 1, tzinfo=UTC),
+ end=datetime(2024, 1, 2, tzinfo=UTC),
+ run_insert=rejected_insert,
+ )
+ assert result.ready is False
+ assert result.capacity_rejected is True
+ assert result.memory_exceeded is False
def test_oom_is_not_retried(self):
calls: list = []
diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py
index 2fcb7ca5027a..c9355d2b2474 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py
@@ -30,6 +30,7 @@ from posthog.hogql.database.database import Database
from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context
+from posthog.exceptions import ClickHouseAtCapacity
from posthog.exceptions_capture import capture_exception
from posthog.hogql_queries.query_runner import QueryRunner
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
@@ -322,8 +323,15 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
if result.ready:
exposure_job_ids = [str(job_id) for job_id in result.job_ids]
self._is_precomputed = True
+ elif result.capacity_rejected:
+ # ClickHouse refused the build because the pool is full. The direct scan
+ # below would hold a slot for minutes and starve the next builds too, so
+ # surface the capacity error and let the caller retry once the pool frees up.
+ raise ClickHouseAtCapacity()
else:
logger.warning("exposure_lazy_computation_not_ready", experiment_id=self.experiment.id)
+ except ClickHouseAtCapacity:
+ raise
except Exception as e:
# Swallowed: the direct-scan fallback below still returns results, which would
# otherwise hide a broken precomputation path. Report so it isn't silent.
@@ -350,13 +358,19 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
numerator_job_ids = [str(job_id) for job_id in ratio_result.numerator.job_ids]
denominator_job_ids = [str(job_id) for job_id in ratio_result.denominator.job_ids]
self._metric_events_precomputed = True
+ elif ratio_result.capacity_rejected:
+ raise ClickHouseAtCapacity()
else:
metric_result = self._ensure_metric_events_precomputed(builder)
if metric_result.ready:
metric_events_job_ids = [str(job_id) for job_id in metric_result.job_ids]
self._metric_events_precomputed = True
+ elif metric_result.capacity_rejected:
+ raise ClickHouseAtCapacity()
if not self._metric_events_precomputed:
logger.warning("metric_events_lazy_computation_not_ready", experiment_id=self.experiment.id)
+ except ClickHouseAtCapacity:
+ raise
except Exception as e:
capture_exception(
e,
diff --git a/products/experiments/backend/hogql_queries/metric_events_precompute.py b/products/experiments/backend/hogql_queries/metric_events_precompute.py
index a39d44565080..e4b5cef00123 100644
--- a/products/experiments/backend/hogql_queries/metric_events_precompute.py
+++ b/products/experiments/backend/hogql_queries/metric_events_precompute.py
@@ -59,6 +59,10 @@ class RatioPrecomputeResults:
def ready(self) -> bool:
return self.numerator.ready and self.denominator.ready
+ @property
+ def capacity_rejected(self) -> bool:
+ return self.numerator.capacity_rejected or self.denominator.capacity_rejected
+
@frozen
class MetricEventsPrecompute:
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
index 3cf26c0b7639..a6e3c53c6809 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
@@ -3,7 +3,7 @@ from datetime import UTC, datetime
from typing import Any, cast
from posthog.test.base import _create_event, _create_person
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
from django.forms.models import model_to_dict
from django.test import override_settings
@@ -24,9 +24,11 @@ from posthog.schema import (
from posthog.hogql.constants import MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY, get_default_hogql_global_settings
from posthog.clickhouse.client import sync_execute
+from posthog.exceptions import ClickHouseAtCapacity
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import (
+ LazyComputationResult,
LazyComputationTable,
_get_insert_settings,
ensure_precomputed,
@@ -916,7 +918,9 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):
patch.object(ExperimentQueryRunner, failing_method, side_effect=Exception("boom")),
# The non-failing path returns "not ready" so it neither errors nor hits ClickHouse,
# keeping the assertion scoped to the single failure under test.
- patch.object(ExperimentQueryRunner, other_method, return_value=MagicMock(ready=False)),
+ patch.object(
+ ExperimentQueryRunner, other_method, return_value=LazyComputationResult(ready=False, job_ids=[])
+ ),
patch(
"products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception"
) as mock_capture,
@@ -932,3 +936,40 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):
# be surfaced to error tracking rather than silently masked.
mock_capture.assert_called_once()
assert mock_capture.call_args.kwargs["additional_properties"]["precomputation_path"] == expected_path
+
+ @parameterized.expand(
+ [
+ ("_ensure_exposures_precomputed", "_ensure_metric_events_precomputed"),
+ ("_ensure_metric_events_precomputed", "_ensure_exposures_precomputed"),
+ ]
+ )
+ def test_raises_at_capacity_instead_of_falling_back_when_build_is_rejected(self, rejected_method, other_method):
+ feature_flag = self.create_feature_flag()
+ experiment = self.create_experiment(
+ feature_flag=feature_flag,
+ start_date=datetime(2024, 1, 1),
+ end_date=datetime(2024, 1, 5),
+ )
+ self._enable_precomputation()
+ metric = ExperimentFunnelMetric(series=[EventsNode(event="purchase")])
+ experiment.metrics = [metric.model_dump(mode="json")]
+ experiment.save()
+
+ with (
+ patch.object(
+ ExperimentQueryRunner,
+ rejected_method,
+ return_value=LazyComputationResult(ready=False, job_ids=[], capacity_rejected=True),
+ ),
+ patch.object(
+ ExperimentQueryRunner, other_method, return_value=LazyComputationResult(ready=False, job_ids=[])
+ ),
+ patch(
+ "products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception"
+ ) as mock_capture,
+ self.assertRaises(ClickHouseAtCapacity),
+ ):
+ self._run_experiment(experiment, metric)
+
+ # A full pool is a retry signal, not a broken precompute path.
+ mock_capture.assert_not_called()