Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): run flagged recalculation queries as the experiments clickhouse user (#119921)

Rodrigo Iloro3511f05df610
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.py
index b07477421496..9523cd09b282 100644
--- a/posthog/clickhouse/client/execute.py
+++ b/posthog/clickhouse/client/execute.py
@@ -488,7 +488,8 @@ def sync_execute(
elif tags.kind == "request" and "api/" in tags_id and "capture" not in tags_id:
# process requests made to API from the PH app
ch_user = ClickHouseUser.APP
- elif tags.feature == Feature.CACHE_WARMUP:
+ # A ch_user tag wins, so warming work such as experiment recalculation can run as its own user.
+ elif tags.feature == Feature.CACHE_WARMUP and not tags.ch_user:
ch_user = ClickHouseUser.CACHE_WARMUP
# update tags if inside temporal (should not). Only meaningful inside a Temporal activity,
diff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.py
index 0f3fa35935ee..375be0a5af89 100644
--- a/posthog/clickhouse/client/test/test_execute.py
+++ b/posthog/clickhouse/client/test/test_execute.py
@@ -118,15 +118,19 @@ def test_llm_analytics_ch_user_routing(client_from_pool, product, kind, tag_id,
@pytest.mark.parametrize(
- "tagged_ch_user,requested_ch_user,expected_ch_user",
+ "tagged_ch_user,feature,requested_ch_user,expected_ch_user",
[
- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),
- (None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),
- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, ClickHouseUser.META, ClickHouseUser.META),
+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),
+ (None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),
+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.META, ClickHouseUser.META),
+ (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),
+ (None, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),
],
)
-def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, requested_ch_user, expected_ch_user):
- with tags_context(product=Product.PRODUCT_ANALYTICS, kind="temporal", id="alerts", ch_user=tagged_ch_user):
+def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, feature, requested_ch_user, expected_ch_user):
+ with tags_context(
+ product=Product.PRODUCT_ANALYTICS, feature=feature, kind="temporal", id="alerts", ch_user=tagged_ch_user
+ ):
sync_execute("SELECT 1", flush=False, ch_user=requested_ch_user)
assert client_from_pool.call_args[0][3] == expected_ch_user
diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.py
index 3d6fe0695627..7499e311fec8 100644
--- a/products/experiments/backend/recalculation.py
+++ b/products/experiments/backend/recalculation.py
@@ -22,7 +22,7 @@ from prometheus_client import Counter
from rest_framework.exceptions import Throttled, ValidationError
from posthog.clickhouse.client import sync_execute
-from posthog.clickhouse.client.connection import Workload
+from posthog.clickhouse.client.connection import ClickHouseUser, Workload
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.event_usage import groups
from posthog.exceptions_capture import capture_exception
@@ -61,6 +61,7 @@ _STALE_RECALC_THRESHOLD = timedelta(minutes=30)
MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)
RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"
RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG = "experiment-recalculation-activity-queue"
+RECALCULATION_CLICKHOUSE_USER_FEATURE_FLAG = "experiment-recalculation-clickhouse-user"
_RATE_LIMITED_TRIGGERS = frozenset(
{
ExperimentMetricsRecalculation.Trigger.MANUAL,
@@ -347,6 +348,16 @@ def _activity_task_queue(organization_id: str) -> str | None:
return settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE if enabled else None
+def recalculation_clickhouse_user(organization_id: str) -> ClickHouseUser | None:
+ # None keeps today's routing, also on a flag error, because that routing always works.
+ try:
+ enabled = _organization_flag_enabled(RECALCULATION_CLICKHOUSE_USER_FEATURE_FLAG, organization_id)
+ except Exception as e:
+ capture_exception(e)
+ return None
+ return ClickHouseUser.EXPERIMENTS if enabled else None
+
+
def _refresh_window_enforced(experiment: Experiment) -> bool:
# Local development skips the window so a developer can reload at will. Tests keep it, because they assert it.
if settings.DEBUG and not settings.TEST:
diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.py
index dbeb98a87d8c..5267988b4454 100644
--- a/products/experiments/backend/temporal/recalculation_logic.py
+++ b/products/experiments/backend/temporal/recalculation_logic.py
@@ -24,7 +24,7 @@ from posthog.schema import ExperimentQuery
from posthog.clickhouse.cancel import cancel_query_on_cluster
from posthog.clickhouse.client.connection import Workload
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
-from posthog.clickhouse.query_tagging import Feature, Product, tag_queries, tags_context
+from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.event_usage import groups
from posthog.exceptions import ClickHouseAtCapacity
from posthog.exceptions_capture import capture_exception
@@ -742,16 +742,21 @@ def _calculate_experiment_metric_for_recalculation_sync(
max_execution_time=METRIC_CALC_MAX_EXECUTION_TIME_SECONDS,
)
+ # Deferred: recalculation.py imports temporal.models, whose package __init__ imports this module.
+ from products.experiments.backend.recalculation import recalculation_clickhouse_user # noqa: PLC0415
+
# Attribute CH load back to this team + product so query_log analysis can tell whose recalc is
# expensive without reverse-engineering the trigger string.
- tag_queries(
+ with tags_context(
trigger="warming/experiment_metrics_recalculation",
team_id=state.team_id,
product=Product.EXPERIMENTS,
feature=Feature.CACHE_WARMUP,
client_query_id=client_query_id,
- )
- result = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
+ # Also covers the precompute inserts, which run inside this context.
+ ch_user=recalculation_clickhouse_user(str(experiment.team.organization_id)),
+ ):
+ result = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)
result_dict = sanitize_non_finite(result.model_dump(mode="json"))
with transaction.atomic():
diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.py
index 0facc1710038..fd8d704c9ee6 100644
--- a/products/experiments/backend/temporal/test_recalculation_activities.py
+++ b/products/experiments/backend/temporal/test_recalculation_activities.py
@@ -9,7 +9,7 @@ from types import SimpleNamespace
import pytest
import time_machine
from posthog.test.base import BaseTest
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
from django.utils import timezone
@@ -19,8 +19,10 @@ from rest_framework.exceptions import ValidationError
from temporalio.exceptions import ApplicationError
from temporalio.testing import ActivityEnvironment
+from posthog.clickhouse.client import sync_execute
+from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
-from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags
+from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.temporal.common.errors import NonReportableError
@@ -1110,6 +1112,35 @@ class TestCalculateActivity(BaseTest):
row = ExperimentMetricResult.objects.get(experiment=exp, metric_uuid="m1")
assert row.query_id == f"experiment_metric_recalc_{recalc.id}_m1_attempt01"
+ @parameterized.expand(
+ [
+ ("flag_on", {"return_value": True}, ClickHouseUser.EXPERIMENTS),
+ ("flag_off", {"return_value": False}, ClickHouseUser.CACHE_WARMUP),
+ ("flag_error", {"side_effect": RuntimeError("flags unavailable")}, ClickHouseUser.CACHE_WARMUP),
+ ]
+ )
+ def test_queries_run_as_the_flagged_clickhouse_user(self, name: str, flag: dict, expected_user: ClickHouseUser):
+ exp = self._experiment(flag_key=f"calc-ch-user-{name}", metrics=[_mean_metric("m1")])
+ recalc = self._recalc(exp, metric_uuids=["m1"])
+
+ def run_metric_and_precompute_queries(**kwargs):
+ sync_execute("SELECT 1", flush=False)
+ # The precompute insert runs inside its own tags_context, as run_lazy_computation_insert does.
+ with tags_context(client_query_id="precompute-job"):
+ sync_execute("SELECT 2", flush=False)
+ return MagicMock(model_dump=MagicMock(return_value={}))
+
+ with (
+ patch("products.experiments.backend.recalculation.feature_enabled_or_false", **flag),
+ patch("posthog.clickhouse.client.execute.get_client_from_pool") as client_from_pool,
+ patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ ):
+ client_from_pool.return_value.__enter__.return_value.execute.return_value = []
+ mock_runner.return_value.run.side_effect = run_metric_and_precompute_queries
+ _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
+
+ assert [call.args[3] for call in client_from_pool.call_args_list] == [expected_user, expected_user]
+
def test_multiple_failures_accumulate_in_metric_errors(self):
# Two metrics fail in sequence (not in parallel — that would need threads + a real Postgres). Pins the
# merge-into-dict behavior: each failure writes its own entry keyed by metric_uuid, no overwriting.