:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): run recalculation calcs on the slot-sized thread pool (#125251)
Rodrigo Ilorob9bdbfccfc15
diff --git a/products/experiments/backend/temporal/recalculation_activities.py b/products/experiments/backend/temporal/recalculation_activities.pyindex 372f759ba3b4..34b27caf5790 100644--- a/products/experiments/backend/temporal/recalculation_activities.py+++ b/products/experiments/backend/temporal/recalculation_activities.py@@ -11,6 +11,7 @@ import structlogimport temporalio.activityfrom posthog.temporal.common.heartbeat import Heartbeater+from posthog.temporal.common.utils import on_asyncify_poolfrom products.experiments.backend.temporal.models import (MAX_METRIC_ATTEMPTS,@@ -128,7 +129,7 @@ async def calculate_experiment_metric_for_recalculation(# cancel, or a timeout the server enforces) only in a heartbeat response, so the drain below depends on it.async with Heartbeater(factor=_HEARTBEAT_FACTOR):task = asyncio.ensure_future(- _calculate_experiment_metric_for_recalculation_sync(+ on_asyncify_pool(_calculate_experiment_metric_for_recalculation_sync)(experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt))diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.pyindex bc3bbdd3eabc..f0b8523170d8 100644--- a/products/experiments/backend/temporal/recalculation_logic.py+++ b/products/experiments/backend/temporal/recalculation_logic.py@@ -71,7 +71,7 @@ _MAX_ERROR_MESSAGE_LENGTH = 2000_OUT_OF_TIME_MESSAGE = "The calculation ran out of time."-# Calculations can fill the event loop's default executor, so a query kill there waits behind the query it kills.+# A query kill on the pool that runs the calculations would wait behind the query it kills._CANCELLATION_EXECUTOR = ThreadPoolExecutor(max_workers=10, thread_name_prefix="experiment-recalc-cancel")# ---------------------------------------------------------------------------@@ -608,7 +608,6 @@ def _cancel_metric_query_sync(recalculation_id: str, metric_uuid: str, attempt:cancel_query_on_cluster(state.team_id, _client_query_id(recalculation_id, metric_uuid, attempt))-@database_sync_to_async_pooldef _calculate_experiment_metric_for_recalculation_sync(experiment_id: int,metric_uuid: str,diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.pyindex cd419ecf1b4f..756986649959 100644--- a/products/experiments/backend/temporal/test_recalculation_activities.py+++ b/products/experiments/backend/temporal/test_recalculation_activities.py@@ -26,6 +26,7 @@ from posthog.clickhouse.client.limit import ConcurrencyLimitExceededfrom posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_contextfrom posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOutfrom posthog.temporal.common.errors import NonReportableError+from posthog.temporal.tests.utils.asyncify import restored_asyncify_executorfrom products.experiments.backend.metric_calculation.config import get_metric_calculation_configfrom products.experiments.backend.metric_calculation.results import _recalc_fingerprint@@ -58,7 +59,6 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag_discover_raw = _discover_experiment_metrics_sync.func # type: ignore[attr-defined]_update_raw = _update_recalculation_progress_sync.func # type: ignore[attr-defined]-_calculate_raw = _calculate_experiment_metric_for_recalculation_sync.func # type: ignore[attr-defined]_cancel_raw = _cancel_metric_query_sync.func # type: ignore[attr-defined]_record_out_of_time_raw = _record_metric_out_of_time_failure_sync.func # type: ignore[attr-defined]@@ -94,7 +94,9 @@ def _calculate(attempt: int = 1,):with patch("products.experiments.backend.temporal.recalculation_logic.close_old_connections"):- return _calculate_raw(experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt)+ return _calculate_experiment_metric_for_recalculation_sync(+ experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt+ )@pytest.mark.django_db(transaction=True)@@ -1445,6 +1447,7 @@ class TestCalculateActivityCancellation:"products.experiments.backend.temporal.recalculation_activities._calculate_experiment_metric_for_recalculation_sync",body,),+ patch("products.experiments.backend.temporal.recalculation_activities.on_asyncify_pool", lambda fn: fn),patch("products.experiments.backend.temporal.recalculation_activities._cancel_metric_query_sync", cancel_query),@@ -1595,6 +1598,36 @@ class TestCalculateActivityCancellation:assert killed == ["experiment_metric_recalc_r1_m1_attempt01"]+ async def test_calculation_runs_while_the_default_executor_is_full(self) -> None:+ # The default executor has node CPUs + 4 threads, so calculations there run fewer at once than the worker's slots.+ default_executor_full = threading.Event()+ busy_default_executor = ThreadPoolExecutor(max_workers=1)+ asyncio.get_running_loop().set_default_executor(busy_default_executor)+ busy_default_executor.submit(default_executor_full.wait)+ try:+ with (+ restored_asyncify_executor() as configure_asyncify_executor,+ patch("products.experiments.backend.temporal.recalculation_logic.close_old_connections"),+ patch(+ "products.experiments.backend.temporal.recalculation_activities._cancel_metric_query_sync",+ AsyncMock(),+ ),+ patch(+ "products.experiments.backend.temporal.recalculation_activities.METRIC_CALC_ACTIVITY_TIMEOUT_SECONDS",+ 0,+ ),+ ):+ configure_asyncify_executor(1)+ with pytest.raises(ApplicationError, match="not a valid ISO datetime"):+ await asyncio.wait_for(+ ActivityEnvironment().run(+ calculate_experiment_metric_for_recalculation, 1, "m1", "r1", "not-a-date"+ ),+ timeout=5,+ )+ finally:+ default_executor_full.set()+async def test_heartbeats_while_the_body_runs_and_stops_after(self) -> None:# Temporal delivers a cancellation only in a heartbeat response, so without heartbeats the drain above# never runs for a workflow cancel or a server-side timeout.diff --git a/products/experiments/backend/temporal/test_recalculation_workflow.py b/products/experiments/backend/temporal/test_recalculation_workflow.pyindex 409767b0f4d7..d0bb61a4560f 100644--- a/products/experiments/backend/temporal/test_recalculation_workflow.py+++ b/products/experiments/backend/temporal/test_recalculation_workflow.py@@ -280,7 +280,7 @@ class TestExperimentMetricsRecalculationWorkflow:async def mock_update_progress(update: RecalculationProgressUpdate) -> str | None:return _START_QUERY_TO if update.mark_started else None- async def failing_sync(+ def failing_sync(experiment_id: int,metric_uuid: str,recalculation_id: str,@@ -393,6 +393,7 @@ class TestExperimentMetricsRecalculationWorkflow:task_queue = str(uuid.uuid4())with (patch(f"{_ACTIVITIES_MODULE}._calculate_experiment_metric_for_recalculation_sync", new=slow_sync),+ patch(f"{_ACTIVITIES_MODULE}.on_asyncify_pool", lambda fn: fn),patch(f"{_ACTIVITIES_MODULE}._cancel_metric_query_sync", new=fake_kill),patch(f"{_WORKFLOW_MODULE}.METRIC_CALC_HEARTBEAT_TIMEOUT_SECONDS", 8),):