Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): run recalculation calcs on the slot-sized thread pool (#125251)

Rodrigo Ilorob9bdbfccfc15
diff --git a/products/experiments/backend/temporal/recalculation_activities.py b/products/experiments/backend/temporal/recalculation_activities.py
index 372f759ba3b4..34b27caf5790 100644
--- a/products/experiments/backend/temporal/recalculation_activities.py
+++ b/products/experiments/backend/temporal/recalculation_activities.py
@@ -11,6 +11,7 @@ import structlog
import temporalio.activity
from posthog.temporal.common.heartbeat import Heartbeater
+from posthog.temporal.common.utils import on_asyncify_pool
from products.experiments.backend.temporal.models import (
MAX_METRIC_ATTEMPTS,
@@ -128,7 +129,7 @@ async def calculate_experiment_metric_for_recalculation(
# cancel, or a timeout the server enforces) only in a heartbeat response, so the drain below depends on it.
async with Heartbeater(factor=_HEARTBEAT_FACTOR):
task = asyncio.ensure_future(
- _calculate_experiment_metric_for_recalculation_sync(
+ on_asyncify_pool(_calculate_experiment_metric_for_recalculation_sync)(
experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt
)
)
diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.py
index bc3bbdd3eabc..f0b8523170d8 100644
--- a/products/experiments/backend/temporal/recalculation_logic.py
+++ b/products/experiments/backend/temporal/recalculation_logic.py
@@ -71,7 +71,7 @@ _MAX_ERROR_MESSAGE_LENGTH = 2000
_OUT_OF_TIME_MESSAGE = "The calculation ran out of time."
-# Calculations can fill the event loop's default executor, so a query kill there waits behind the query it kills.
+# A query kill on the pool that runs the calculations would wait behind the query it kills.
_CANCELLATION_EXECUTOR = ThreadPoolExecutor(max_workers=10, thread_name_prefix="experiment-recalc-cancel")
# ---------------------------------------------------------------------------
@@ -608,7 +608,6 @@ def _cancel_metric_query_sync(recalculation_id: str, metric_uuid: str, attempt:
cancel_query_on_cluster(state.team_id, _client_query_id(recalculation_id, metric_uuid, attempt))
-@database_sync_to_async_pool
def _calculate_experiment_metric_for_recalculation_sync(
experiment_id: int,
metric_uuid: str,
diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.py
index cd419ecf1b4f..756986649959 100644
--- a/products/experiments/backend/temporal/test_recalculation_activities.py
+++ b/products/experiments/backend/temporal/test_recalculation_activities.py
@@ -26,6 +26,7 @@ from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.temporal.common.errors import NonReportableError
+from posthog.temporal.tests.utils.asyncify import restored_asyncify_executor
from products.experiments.backend.metric_calculation.config import get_metric_calculation_config
from products.experiments.backend.metric_calculation.results import _recalc_fingerprint
@@ -58,7 +59,6 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag
_discover_raw = _discover_experiment_metrics_sync.func # type: ignore[attr-defined]
_update_raw = _update_recalculation_progress_sync.func # type: ignore[attr-defined]
-_calculate_raw = _calculate_experiment_metric_for_recalculation_sync.func # type: ignore[attr-defined]
_cancel_raw = _cancel_metric_query_sync.func # type: ignore[attr-defined]
_record_out_of_time_raw = _record_metric_out_of_time_failure_sync.func # type: ignore[attr-defined]
@@ -94,7 +94,9 @@ def _calculate(
attempt: int = 1,
):
with patch("products.experiments.backend.temporal.recalculation_logic.close_old_connections"):
- return _calculate_raw(experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt)
+ return _calculate_experiment_metric_for_recalculation_sync(
+ experiment_id, metric_uuid, recalculation_id, query_to, role, is_final_attempt, attempt
+ )
@pytest.mark.django_db(transaction=True)
@@ -1445,6 +1447,7 @@ class TestCalculateActivityCancellation:
"products.experiments.backend.temporal.recalculation_activities._calculate_experiment_metric_for_recalculation_sync",
body,
),
+ patch("products.experiments.backend.temporal.recalculation_activities.on_asyncify_pool", lambda fn: fn),
patch(
"products.experiments.backend.temporal.recalculation_activities._cancel_metric_query_sync", cancel_query
),
@@ -1595,6 +1598,36 @@ class TestCalculateActivityCancellation:
assert killed == ["experiment_metric_recalc_r1_m1_attempt01"]
+ async def test_calculation_runs_while_the_default_executor_is_full(self) -> None:
+ # The default executor has node CPUs + 4 threads, so calculations there run fewer at once than the worker's slots.
+ default_executor_full = threading.Event()
+ busy_default_executor = ThreadPoolExecutor(max_workers=1)
+ asyncio.get_running_loop().set_default_executor(busy_default_executor)
+ busy_default_executor.submit(default_executor_full.wait)
+ try:
+ with (
+ restored_asyncify_executor() as configure_asyncify_executor,
+ patch("products.experiments.backend.temporal.recalculation_logic.close_old_connections"),
+ patch(
+ "products.experiments.backend.temporal.recalculation_activities._cancel_metric_query_sync",
+ AsyncMock(),
+ ),
+ patch(
+ "products.experiments.backend.temporal.recalculation_activities.METRIC_CALC_ACTIVITY_TIMEOUT_SECONDS",
+ 0,
+ ),
+ ):
+ configure_asyncify_executor(1)
+ with pytest.raises(ApplicationError, match="not a valid ISO datetime"):
+ await asyncio.wait_for(
+ ActivityEnvironment().run(
+ calculate_experiment_metric_for_recalculation, 1, "m1", "r1", "not-a-date"
+ ),
+ timeout=5,
+ )
+ finally:
+ default_executor_full.set()
+
async def test_heartbeats_while_the_body_runs_and_stops_after(self) -> None:
# Temporal delivers a cancellation only in a heartbeat response, so without heartbeats the drain above
# never runs for a workflow cancel or a server-side timeout.
diff --git a/products/experiments/backend/temporal/test_recalculation_workflow.py b/products/experiments/backend/temporal/test_recalculation_workflow.py
index 409767b0f4d7..d0bb61a4560f 100644
--- a/products/experiments/backend/temporal/test_recalculation_workflow.py
+++ b/products/experiments/backend/temporal/test_recalculation_workflow.py
@@ -280,7 +280,7 @@ class TestExperimentMetricsRecalculationWorkflow:
async def mock_update_progress(update: RecalculationProgressUpdate) -> str | None:
return _START_QUERY_TO if update.mark_started else None
- async def failing_sync(
+ def failing_sync(
experiment_id: int,
metric_uuid: str,
recalculation_id: str,
@@ -393,6 +393,7 @@ class TestExperimentMetricsRecalculationWorkflow:
task_queue = str(uuid.uuid4())
with (
patch(f"{_ACTIVITIES_MODULE}._calculate_experiment_metric_for_recalculation_sync", new=slow_sync),
+ patch(f"{_ACTIVITIES_MODULE}.on_asyncify_pool", lambda fn: fn),
patch(f"{_ACTIVITIES_MODULE}._cancel_metric_query_sync", new=fake_kill),
patch(f"{_WORKFLOW_MODULE}.METRIC_CALC_HEARTBEAT_TIMEOUT_SECONDS", 8),
):