Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): use default load balancing for metric events precompute (#123713)

Rodrigo Iloro0f5760603d4e
diff --git a/posthog/hogql/database/schema/experiment_metric_events_preaggregated.py b/posthog/hogql/database/schema/experiment_metric_events_preaggregated.py
index ff5f4acc7417..164ca9b43e4d 100644
--- a/posthog/hogql/database/schema/experiment_metric_events_preaggregated.py
+++ b/posthog/hogql/database/schema/experiment_metric_events_preaggregated.py
@@ -1,6 +1,3 @@
-from pydantic import Field
-
-from posthog.hogql.constants import HogQLQuerySettings
from posthog.hogql.database.models import (
DateTimeDatabaseField,
FieldOrTable,
@@ -18,9 +15,6 @@ class ExperimentMetricEventsPreaggregatedTable(Table):
"Internal preaggregated table of experiment metric events, one row per metric-contributing event recording "
"its numeric value and (for funnel metrics) the steps completed. Powers experiment results computation."
)
- top_level_settings: HogQLQuerySettings | None = Field(
- default_factory=lambda: HogQLQuerySettings(load_balancing="in_order")
- )
fields: dict[str, FieldOrTable] = {
"team_id": IntegerDatabaseField(name="team_id"),
diff --git a/products/analytics_platform/backend/lazy_computation/CONSISTENCY.md b/products/analytics_platform/backend/lazy_computation/CONSISTENCY.md
index 32043c9dafe7..634ba1590e1a 100644
--- a/products/analytics_platform/backend/lazy_computation/CONSISTENCY.md
+++ b/products/analytics_platform/backend/lazy_computation/CONSISTENCY.md
@@ -58,6 +58,20 @@ Only works with ReplicatedMergeTree family tables, not Distributed tables direct
Because of this trade, quorum is opt-out per caller: `ensure_precomputed(read_after_write=False)` drops it for builds whose readers arrive later (background warmers, revalidation tasks). Those callers get nothing from the read-your-own-writes guarantee, so for them quorum is pure availability risk: replicas count against quorum by ZooKeeper registration, not liveness, so a dead-but-registered replica (mid-upscale, unrevived node) fails every quorum INSERT until someone repairs the cluster metadata. `web_ensure_precomputed` defaults `read_after_write` to `False`, so every web analytics build skips quorum unless a caller overrides it. Callers that SELECT the rows in the same request (experiments) keep the framework default of `True`. The residual same-call race (build, then read before the part replicates) is handled with a settle wait: `LazyComputationResult.freshly_built` marks coverage that landed during the call, and `ensure_precomputed` sleeps `PREAGGREGATION_REPLICATION_SETTLE_SECONDS` (1s, above the observed sub-second replica lag) before returning it to a quorum-skipping caller, so the caller's immediate read-back cannot cache a partial result.
+The INSERT takes `load_balancing` from the HogQL table that reads the rows (`_get_insert_settings`), so the write and the read choose replicas the same way. Web analytics, marketing, `preaggregation_results` and experiment exposures tables set `in_order` (Approach E). Experiment exposures live on the sharded `posthog` cluster. When this was written it had three replicas per shard, so `'auto'` quorum waits for two of them and a spread read can reach the third before it has the part. The experiment metric events table lives on the `aux` cluster, which had one shard with two replicas at the time. It sets no `load_balancing` and uses the profile default, so its reads spread over both replicas and rely on quorum alone (Approach B). Approach B is exact only while `'auto'` quorum covers every replica, which means at most two replicas per shard. `load_balancing` applies to the whole query, so a SELECT that joins both experiment tables still runs `in_order`.
+
+The replica counts above are a snapshot, not a guarantee. Check the current topology in each region before relying on them:
+
+```sql
+SELECT cluster, shard_num, count() AS replicas
+FROM system.clusters
+WHERE cluster IN ('aux', 'posthog')
+GROUP BY cluster, shard_num
+ORDER BY cluster, shard_num
+```
+
+The metric events spread assumes `aux` has at most two replicas per shard. Nothing enforces that assumption yet; a separate PR adds the enforcement.
+
Sources:
- [ClickHouse settings docs](https://clickhouse.com/docs/operations/settings/settings#insert_quorum)
diff --git a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
index a9a542609ae5..c9600d064b54 100644
--- a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
+++ b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py
@@ -22,12 +22,24 @@ from prometheus_client import Counter
from posthog.schema import HogQLQueryModifiers
from posthog.hogql import ast
-from posthog.hogql.constants import (
- MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY,
- HogQLQuerySettings,
- get_default_hogql_global_settings,
-)
+from posthog.hogql.constants import MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY, get_default_hogql_global_settings
from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import Table
+from posthog.hogql.database.schema.experiment_exposures_preaggregated import ExperimentExposuresPreaggregatedTable
+from posthog.hogql.database.schema.experiment_metric_events_preaggregated import (
+ ExperimentMetricEventsPreaggregatedTable,
+)
+from posthog.hogql.database.schema.marketing_conversions_preaggregated import MarketingConversionsPreaggregatedTable
+from posthog.hogql.database.schema.marketing_costs_preaggregated import MarketingCostsPreaggregatedTable
+from posthog.hogql.database.schema.marketing_touchpoints_preaggregated import MarketingTouchpointsPreaggregatedTable
+from posthog.hogql.database.schema.preaggregation_results import PreaggregationResultsTable
+from posthog.hogql.database.schema.web_goals_preaggregated import WebGoalsPreaggregatedTable
+from posthog.hogql.database.schema.web_overview_preaggregated import WebOverviewPreaggregatedTable
+from posthog.hogql.database.schema.web_sessions_preaggregated import WebSessionsPreaggregatedTable
+from posthog.hogql.database.schema.web_stats_frustration_preaggregated import WebStatsFrustrationPreaggregatedTable
+from posthog.hogql.database.schema.web_stats_paths_preaggregated import WebStatsPathsPreaggregatedTable
+from posthog.hogql.database.schema.web_stats_preaggregated import WebStatsPreaggregatedTable
+from posthog.hogql.database.schema.web_vitals_paths_preaggregated import WebVitalsPathsPreaggregatedTable
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.parser import parse_select
from posthog.hogql.placeholders import replace_placeholders
@@ -178,6 +190,7 @@ LAZY_COMPUTATION_JOB_CREATE_CONFLICTS_TOTAL = Counter(
def _get_insert_settings(
team_id: int,
+ table: "LazyComputationTable",
*,
spill_to_disk: bool = False,
read_after_write: bool = True,
@@ -199,6 +212,9 @@ def _get_insert_settings(
minutes later (background builders) gets no consistency from it — only the
failure mode where a downed or stale-registered replica turns every build
into TOO_FEW_LIVE_REPLICAS until the cluster is repaired.
+
+ `load_balancing` comes from the HogQL table that reads `table`, so the INSERT and the
+ later SELECT choose replicas the same way. A table that sets none uses the profile default.
"""
settings = get_default_hogql_global_settings(team_id=team_id).model_dump(exclude_none=True)
settings.pop("readonly", None) # INSERTs need write access
@@ -214,7 +230,7 @@ def _get_insert_settings(
# Uses the legacy name of `distributed_foreground_insert` (renamed in ClickHouse 23.x)
# for version compatibility.
"insert_distributed_sync": 1,
- **HogQLQuerySettings(load_balancing="in_order").model_dump(exclude_none=True),
+ **_read_table_load_balancing(table),
}
)
if spill_to_disk:
@@ -592,6 +608,33 @@ class LazyComputationTable(StrEnum):
WEB_SESSIONS_DIMENSIONAL_PREAGGREGATED = "web_sessions_dimensional_preaggregated"
+# The HogQL table that reads each target table. The two dimensional web tables are read
+# without HogQL, so they have no entry.
+_READ_TABLES: dict[LazyComputationTable, Table] = {
+ LazyComputationTable.PREAGGREGATION_RESULTS: PreaggregationResultsTable(),
+ LazyComputationTable.EXPERIMENT_EXPOSURES_PREAGGREGATED: ExperimentExposuresPreaggregatedTable(),
+ LazyComputationTable.EXPERIMENT_METRIC_EVENTS_PREAGGREGATED: ExperimentMetricEventsPreaggregatedTable(),
+ LazyComputationTable.MARKETING_TOUCHPOINTS_PREAGGREGATED: MarketingTouchpointsPreaggregatedTable(),
+ LazyComputationTable.MARKETING_CONVERSIONS_PREAGGREGATED: MarketingConversionsPreaggregatedTable(),
+ LazyComputationTable.MARKETING_COSTS_PREAGGREGATED: MarketingCostsPreaggregatedTable(),
+ LazyComputationTable.WEB_OVERVIEW_PREAGGREGATED: WebOverviewPreaggregatedTable(),
+ LazyComputationTable.WEB_STATS_PREAGGREGATED: WebStatsPreaggregatedTable(),
+ LazyComputationTable.WEB_STATS_PATHS_PREAGGREGATED: WebStatsPathsPreaggregatedTable(),
+ LazyComputationTable.WEB_VITALS_PATHS_PREAGGREGATED: WebVitalsPathsPreaggregatedTable(),
+ LazyComputationTable.WEB_STATS_FRUSTRATION_PREAGGREGATED: WebStatsFrustrationPreaggregatedTable(),
+ LazyComputationTable.WEB_GOALS_PREAGGREGATED: WebGoalsPreaggregatedTable(),
+ LazyComputationTable.WEB_SESSIONS_DIMENSIONAL_PREAGGREGATED: WebSessionsPreaggregatedTable(),
+}
+
+
+def _read_table_load_balancing(table: LazyComputationTable) -> dict[str, str]:
+ read_table = _READ_TABLES.get(table)
+ if read_table is None or read_table.top_level_settings is None:
+ return {}
+ load_balancing = read_table.top_level_settings.load_balancing
+ return {"load_balancing": load_balancing} if load_balancing is not None else {}
+
+
# Tables where expires_at is a Date (not DateTime64). Date truncates to midnight,
# so an expires_at just after midnight would round down to a time *before* the PG
# job expires. We add an extra day of buffer for these tables.
@@ -1081,7 +1124,7 @@ def run_lazy_computation_insert(
sync_execute(
insert_sql,
values,
- settings=_get_insert_settings(team.id),
+ settings=_get_insert_settings(team.id, query_info.table),
)
)
@@ -1876,6 +1919,7 @@ def ensure_precomputed(
values,
settings=_get_insert_settings(
t.id,
+ table,
spill_to_disk=spill_to_disk,
read_after_write=read_after_write,
overrides=insert_settings,
diff --git a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
index c990f26c2b4a..3f9d356d0850 100644
--- a/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
+++ b/products/analytics_platform/backend/lazy_computation/tests/test_lazy_computation_executor.py
@@ -3513,9 +3513,28 @@ class TestIsNonRetryableError(BaseTest):
assert is_non_retryable_error(error) is False
+_INSERT_LOAD_BALANCING_CASES = [
+ (LazyComputationTable.EXPERIMENT_EXPOSURES_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.EXPERIMENT_METRIC_EVENTS_PREAGGREGATED, None),
+ (LazyComputationTable.PREAGGREGATION_RESULTS, "in_order"),
+ (LazyComputationTable.MARKETING_TOUCHPOINTS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.MARKETING_CONVERSIONS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.MARKETING_COSTS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_OVERVIEW_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_STATS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_STATS_PATHS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_VITALS_PATHS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_STATS_FRUSTRATION_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_GOALS_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_SESSIONS_DIMENSIONAL_PREAGGREGATED, "in_order"),
+ (LazyComputationTable.WEB_STATS_DIMENSIONAL_PREAGGREGATED, None),
+ (LazyComputationTable.WEB_BOUNCES_DIMENSIONAL_PREAGGREGATED, None),
+]
+
+
class TestInsertSettings(BaseTest):
def test_insert_settings_guarantee_read_your_writes(self):
- settings = _get_insert_settings(self.team.pk)
+ settings = _get_insert_settings(self.team.pk, LazyComputationTable.PREAGGREGATION_RESULTS)
# The executor marks jobs READY the moment the INSERT returns, which is only sound if
# the distributed write is synchronous. Production profiles default to async, so this
@@ -3525,7 +3544,6 @@ class TestInsertSettings(BaseTest):
# Quorum breakage (dead replica, stale ZK registration) must fail fast and fall back,
# not hold the request for ClickHouse's 600s default quorum wait.
assert settings["insert_quorum_timeout"] == PREAGGREGATION_INSERT_QUORUM_TIMEOUT_MS
- assert settings["load_balancing"] == "in_order"
assert settings["max_execution_time"] == HOGQL_INCREASED_MAX_EXECUTION_TIME
assert "readonly" not in settings
@@ -3538,8 +3556,16 @@ class TestInsertSettings(BaseTest):
f"products.analytics_platform.backend.lazy_computation.lazy_computation_executor.PREAGGREGATION_INSERT_QUORUM",
"auto",
):
- assert _get_insert_settings(self.team.pk)["insert_quorum"] == "auto"
- assert _get_insert_settings(self.team.pk, read_after_write=False)["insert_quorum"] == 0
+ table = LazyComputationTable.PREAGGREGATION_RESULTS
+ assert _get_insert_settings(self.team.pk, table)["insert_quorum"] == "auto"
+ assert _get_insert_settings(self.team.pk, table, read_after_write=False)["insert_quorum"] == 0
+
+ @parameterized.expand(_INSERT_LOAD_BALANCING_CASES)
+ def test_insert_load_balancing_follows_the_read_table(self, table, expected_load_balancing):
+ assert _get_insert_settings(self.team.pk, table).get("load_balancing") == expected_load_balancing
+
+ def test_insert_load_balancing_cases_cover_every_table(self):
+ assert {table for table, _ in _INSERT_LOAD_BALANCING_CASES} == set(LazyComputationTable)
class TestInsertSettingsAppliedToInserts(BaseTest):
@@ -3594,6 +3620,7 @@ class TestInsertSettingsAppliedToInserts(BaseTest):
)
expected_settings = _get_insert_settings(
self.team.pk,
+ LazyComputationTable.PREAGGREGATION_RESULTS,
spill_to_disk=extra_kwargs.get("spill_to_disk", False),
read_after_write=extra_kwargs.get("read_after_write", True),
overrides=extra_kwargs.get("insert_settings"),
@@ -3669,7 +3696,9 @@ class TestInsertSettingsAppliedToInserts(BaseTest):
run_lazy_computation_insert(self.team, job, query_info)
assert mock_execute.call_count == 1
- assert mock_execute.call_args.kwargs["settings"] == _get_insert_settings(self.team.pk)
+ assert mock_execute.call_args.kwargs["settings"] == _get_insert_settings(
+ self.team.pk, LazyComputationTable.PREAGGREGATION_RESULTS
+ )
class TestInsertsReportRowsWritten(BaseTest):
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
index a6e3c53c6809..94c559e65c57 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
@@ -724,7 +724,7 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):
INSERT-specific overrides like readonly, max_execution_time,
insert_quorum, and load_balancing."""
hogql_settings = get_default_hogql_global_settings(team_id=self.team.id).model_dump(exclude_none=True)
- insert_settings = _get_insert_settings(self.team.id)
+ insert_settings = _get_insert_settings(self.team.id, LazyComputationTable.EXPERIMENT_EXPOSURES_PREAGGREGATED)
# These are intentionally different for INSERTs
intentional_overrides = {"readonly", "max_execution_time", "insert_quorum", "load_balancing"}