:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): run flagged recalculation queries as the experiments clickhouse user (#119921)
Rodrigo Iloro3511f05df610
diff --git a/posthog/clickhouse/client/execute.py b/posthog/clickhouse/client/execute.pyindex b07477421496..9523cd09b282 100644--- a/posthog/clickhouse/client/execute.py+++ b/posthog/clickhouse/client/execute.py@@ -488,7 +488,8 @@ def sync_execute(elif tags.kind == "request" and "api/" in tags_id and "capture" not in tags_id:# process requests made to API from the PH appch_user = ClickHouseUser.APP- elif tags.feature == Feature.CACHE_WARMUP:+ # A ch_user tag wins, so warming work such as experiment recalculation can run as its own user.+ elif tags.feature == Feature.CACHE_WARMUP and not tags.ch_user:ch_user = ClickHouseUser.CACHE_WARMUP# update tags if inside temporal (should not). Only meaningful inside a Temporal activity,diff --git a/posthog/clickhouse/client/test/test_execute.py b/posthog/clickhouse/client/test/test_execute.pyindex 0f3fa35935ee..375be0a5af89 100644--- a/posthog/clickhouse/client/test/test_execute.py+++ b/posthog/clickhouse/client/test/test_execute.py@@ -118,15 +118,19 @@ def test_llm_analytics_ch_user_routing(client_from_pool, product, kind, tag_id,@pytest.mark.parametrize(- "tagged_ch_user,requested_ch_user,expected_ch_user",+ "tagged_ch_user,feature,requested_ch_user,expected_ch_user",[- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),- (None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),- (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, ClickHouseUser.META, ClickHouseUser.META),+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.DEFAULT, ClickHouseUser.ALERTS_PLATFORM_INSIGHT),+ (None, None, ClickHouseUser.DEFAULT, ClickHouseUser.DEFAULT),+ (ClickHouseUser.ALERTS_PLATFORM_INSIGHT, None, ClickHouseUser.META, ClickHouseUser.META),+ (ClickHouseUser.EXPERIMENTS, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.EXPERIMENTS),+ (None, Feature.CACHE_WARMUP, ClickHouseUser.DEFAULT, ClickHouseUser.CACHE_WARMUP),],)-def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, requested_ch_user, expected_ch_user):- with tags_context(product=Product.PRODUCT_ANALYTICS, kind="temporal", id="alerts", ch_user=tagged_ch_user):+def test_tagged_ch_user_routing(client_from_pool, tagged_ch_user, feature, requested_ch_user, expected_ch_user):+ with tags_context(+ product=Product.PRODUCT_ANALYTICS, feature=feature, kind="temporal", id="alerts", ch_user=tagged_ch_user+ ):sync_execute("SELECT 1", flush=False, ch_user=requested_ch_user)assert client_from_pool.call_args[0][3] == expected_ch_userdiff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.pyindex 3d6fe0695627..7499e311fec8 100644--- a/products/experiments/backend/recalculation.py+++ b/products/experiments/backend/recalculation.py@@ -22,7 +22,7 @@ from prometheus_client import Counterfrom rest_framework.exceptions import Throttled, ValidationErrorfrom posthog.clickhouse.client import sync_execute-from posthog.clickhouse.client.connection import Workload+from posthog.clickhouse.client.connection import ClickHouseUser, Workloadfrom posthog.clickhouse.query_tagging import Feature, Product, tags_contextfrom posthog.event_usage import groupsfrom posthog.exceptions_capture import capture_exception@@ -61,6 +61,7 @@ _STALE_RECALC_THRESHOLD = timedelta(minutes=30)MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG = "experiment-recalculation-activity-queue"+RECALCULATION_CLICKHOUSE_USER_FEATURE_FLAG = "experiment-recalculation-clickhouse-user"_RATE_LIMITED_TRIGGERS = frozenset({ExperimentMetricsRecalculation.Trigger.MANUAL,@@ -347,6 +348,16 @@ def _activity_task_queue(organization_id: str) -> str | None:return settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE if enabled else None+def recalculation_clickhouse_user(organization_id: str) -> ClickHouseUser | None:+ # None keeps today's routing, also on a flag error, because that routing always works.+ try:+ enabled = _organization_flag_enabled(RECALCULATION_CLICKHOUSE_USER_FEATURE_FLAG, organization_id)+ except Exception as e:+ capture_exception(e)+ return None+ return ClickHouseUser.EXPERIMENTS if enabled else None++def _refresh_window_enforced(experiment: Experiment) -> bool:# Local development skips the window so a developer can reload at will. Tests keep it, because they assert it.if settings.DEBUG and not settings.TEST:diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.pyindex dbeb98a87d8c..5267988b4454 100644--- a/products/experiments/backend/temporal/recalculation_logic.py+++ b/products/experiments/backend/temporal/recalculation_logic.py@@ -24,7 +24,7 @@ from posthog.schema import ExperimentQueryfrom posthog.clickhouse.cancel import cancel_query_on_clusterfrom posthog.clickhouse.client.connection import Workloadfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded-from posthog.clickhouse.query_tagging import Feature, Product, tag_queries, tags_context+from posthog.clickhouse.query_tagging import Feature, Product, tags_contextfrom posthog.event_usage import groupsfrom posthog.exceptions import ClickHouseAtCapacityfrom posthog.exceptions_capture import capture_exception@@ -742,16 +742,21 @@ def _calculate_experiment_metric_for_recalculation_sync(max_execution_time=METRIC_CALC_MAX_EXECUTION_TIME_SECONDS,)+ # Deferred: recalculation.py imports temporal.models, whose package __init__ imports this module.+ from products.experiments.backend.recalculation import recalculation_clickhouse_user # noqa: PLC0415+# Attribute CH load back to this team + product so query_log analysis can tell whose recalc is# expensive without reverse-engineering the trigger string.- tag_queries(+ with tags_context(trigger="warming/experiment_metrics_recalculation",team_id=state.team_id,product=Product.EXPERIMENTS,feature=Feature.CACHE_WARMUP,client_query_id=client_query_id,- )- result = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)+ # Also covers the precompute inserts, which run inside this context.+ ch_user=recalculation_clickhouse_user(str(experiment.team.organization_id)),+ ):+ result = runner.run(execution_mode=ExecutionMode.CALCULATE_BLOCKING_ALWAYS)result_dict = sanitize_non_finite(result.model_dump(mode="json"))with transaction.atomic():diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.pyindex 0facc1710038..fd8d704c9ee6 100644--- a/products/experiments/backend/temporal/test_recalculation_activities.py+++ b/products/experiments/backend/temporal/test_recalculation_activities.py@@ -9,7 +9,7 @@ from types import SimpleNamespaceimport pytestimport time_machinefrom posthog.test.base import BaseTest-from unittest.mock import AsyncMock, patch+from unittest.mock import AsyncMock, MagicMock, patchfrom django.utils import timezone@@ -19,8 +19,10 @@ from rest_framework.exceptions import ValidationErrorfrom temporalio.exceptions import ApplicationErrorfrom temporalio.testing import ActivityEnvironment+from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import ClickHouseUserfrom posthog.clickhouse.client.limit import ConcurrencyLimitExceeded-from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags+from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_contextfrom posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOutfrom posthog.temporal.common.errors import NonReportableError@@ -1110,6 +1112,35 @@ class TestCalculateActivity(BaseTest):row = ExperimentMetricResult.objects.get(experiment=exp, metric_uuid="m1")assert row.query_id == f"experiment_metric_recalc_{recalc.id}_m1_attempt01"+ @parameterized.expand(+ [+ ("flag_on", {"return_value": True}, ClickHouseUser.EXPERIMENTS),+ ("flag_off", {"return_value": False}, ClickHouseUser.CACHE_WARMUP),+ ("flag_error", {"side_effect": RuntimeError("flags unavailable")}, ClickHouseUser.CACHE_WARMUP),+ ]+ )+ def test_queries_run_as_the_flagged_clickhouse_user(self, name: str, flag: dict, expected_user: ClickHouseUser):+ exp = self._experiment(flag_key=f"calc-ch-user-{name}", metrics=[_mean_metric("m1")])+ recalc = self._recalc(exp, metric_uuids=["m1"])++ def run_metric_and_precompute_queries(**kwargs):+ sync_execute("SELECT 1", flush=False)+ # The precompute insert runs inside its own tags_context, as run_lazy_computation_insert does.+ with tags_context(client_query_id="precompute-job"):+ sync_execute("SELECT 2", flush=False)+ return MagicMock(model_dump=MagicMock(return_value={}))++ with (+ patch("products.experiments.backend.recalculation.feature_enabled_or_false", **flag),+ patch("posthog.clickhouse.client.execute.get_client_from_pool") as client_from_pool,+ patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ ):+ client_from_pool.return_value.__enter__.return_value.execute.return_value = []+ mock_runner.return_value.run.side_effect = run_metric_and_precompute_queries+ _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)++ assert [call.args[3] for call in client_from_pool.call_args_list] == [expected_user, expected_user]+def test_multiple_failures_accumulate_in_metric_errors(self):# Two metrics fail in sequence (not in parallel — that would need threads + a real Postgres). Pins the# merge-into-dict behavior: each failure writes its own entry keyed by metric_uuid, no overwriting.