Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): route metric calculation to the activity task queue (#118931)

Rodrigo Ilorof135e86c7d56
diff --git a/products/experiments/backend/temporal/recalculation_workflow.py b/products/experiments/backend/temporal/recalculation_workflow.py
index 3d845b6580a8..c2af3789cb89 100644
--- a/products/experiments/backend/temporal/recalculation_workflow.py
+++ b/products/experiments/backend/temporal/recalculation_workflow.py
@@ -157,6 +157,13 @@ class ExperimentMetricsRecalculationWorkflow(PostHogWorkflow):
maximum_interval=timedelta(seconds=RECALCULATION_RETRY_MAX_INTERVAL_SECONDS),
maximum_attempts=MAX_METRIC_ATTEMPTS,
)
+ # Pre-patch histories scheduled every calculation on the workflow's queue, even when an old worker
+ # received a run with the field set, so they must replay with None.
+ activity_task_queue = (
+ inputs.activity_task_queue
+ if inputs.activity_task_queue and temporalio.workflow.patched("recalc-activity-task-queue-2026-10")
+ else None
+ )
results = await asyncio.gather(
*[
temporalio.workflow.execute_activity(
@@ -172,6 +179,7 @@ class ExperimentMetricsRecalculationWorkflow(PostHogWorkflow):
schedule_to_close_timeout=timedelta(seconds=METRIC_CALC_SCHEDULE_TO_CLOSE_TIMEOUT_SECONDS),
heartbeat_timeout=timedelta(seconds=METRIC_CALC_HEARTBEAT_TIMEOUT_SECONDS),
retry_policy=calc_retry_policy,
+ task_queue=activity_task_queue,
# Round-robin dispatch across orgs under backlog; a no-op on namespaces without
# fairness support.
priority=Priority(fairness_key=inputs.fairness_key),
diff --git a/products/experiments/backend/temporal/test_recalculation_workflow.py b/products/experiments/backend/temporal/test_recalculation_workflow.py
index 3cda8c2b7686..409767b0f4d7 100644
--- a/products/experiments/backend/temporal/test_recalculation_workflow.py
+++ b/products/experiments/backend/temporal/test_recalculation_workflow.py
@@ -8,13 +8,16 @@ import pytest
from unittest.mock import patch
import temporalio.worker
+import temporalio.workflow
from parameterized import parameterized
from temporalio import activity
-from temporalio.client import WorkflowFailureError
+from temporalio.client import WorkflowFailureError, WorkflowHistory
from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
+from posthog.dataclasses import frozen
+
from products.experiments.backend.recalculation import _STALE_RECALC_THRESHOLD
from products.experiments.backend.temporal.models import (
MAX_METRIC_ATTEMPTS,
@@ -118,6 +121,69 @@ async def _run_workflow(activities) -> dict:
)
+def _recording_calculate(calls: list[str]) -> Callable:
+ @activity.defn(name="calculate_experiment_metric_for_recalculation")
+ async def mock_calculate(
+ experiment_id: int,
+ metric_uuid: str,
+ recalculation_id: str,
+ query_to: str,
+ role: str = "primary",
+ ) -> MetricRecalculationResult:
+ calls.append(metric_uuid)
+ return MetricRecalculationResult(metric_uuid=metric_uuid, success=True)
+
+ return mock_calculate
+
+
+@frozen
+class _TwoQueueRun:
+ result: dict
+ workflow_queue_calls: list[str]
+ activity_queue_calls: list[str]
+ history: WorkflowHistory
+
+
+async def _run_with_activity_queue_worker(
+ activity_task_queue: str | None, activity_worker_task_queue: str
+) -> _TwoQueueRun:
+ activities, _progress, _calls = _make_mock_activities(metrics=[_metric("m1")])
+ workflow_queue_calls: list[str] = []
+ activity_queue_calls: list[str] = []
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with (
+ Worker(
+ env.client,
+ task_queue=task_queue,
+ workflows=[ExperimentMetricsRecalculationWorkflow],
+ activities=[*activities[:2], _recording_calculate(workflow_queue_calls)],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ),
+ Worker(
+ env.client,
+ task_queue=activity_worker_task_queue,
+ activities=[_recording_calculate(activity_queue_calls)],
+ ),
+ ):
+ handle = await env.client.start_workflow(
+ ExperimentMetricsRecalculationWorkflow.run,
+ ExperimentMetricsRecalculationWorkflowInputs(
+ recalculation_id=str(uuid.uuid4()), activity_task_queue=activity_task_queue
+ ),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ )
+ result = await handle.result()
+ history = await handle.fetch_history()
+ return _TwoQueueRun(
+ result=result,
+ workflow_queue_calls=workflow_queue_calls,
+ activity_queue_calls=activity_queue_calls,
+ history=history,
+ )
+
+
@pytest.mark.asyncio
class TestExperimentMetricsRecalculationWorkflow:
async def test_no_metrics_completes_immediately(self):
@@ -554,3 +620,49 @@ class TestExperimentMetricsRecalculationWorkflow:
with pytest.raises(WorkflowFailureError):
await _run_workflow([mock_discover, mock_update_progress, mock_calculate])
+
+ @parameterized.expand(
+ [
+ # name, set_field, expected_workflow_queue_calls, expected_activity_queue_calls
+ ("field_unset_stays_on_workflow_queue", False, ["m1"], []),
+ ("field_set_routes_to_activity_queue", True, [], ["m1"]),
+ ]
+ )
+ async def test_activity_task_queue_routes_the_calculation(
+ self,
+ name: str,
+ set_field: bool,
+ expected_workflow_queue_calls: list[str],
+ expected_activity_queue_calls: list[str],
+ ):
+ activity_worker_task_queue = str(uuid.uuid4())
+ run = await _run_with_activity_queue_worker(
+ activity_task_queue=activity_worker_task_queue if set_field else None,
+ activity_worker_task_queue=activity_worker_task_queue,
+ )
+
+ assert run.result == {"total": 1, "succeeded": 1, "failed": 0}
+ assert run.workflow_queue_calls == expected_workflow_queue_calls
+ assert run.activity_queue_calls == expected_activity_queue_calls
+
+ async def test_pre_patch_history_with_the_field_set_replays(self):
+ # An old worker ignores the unknown input field and records no patch marker, so the new code must
+ # replay that history with the calculation on the workflow's queue.
+ activity_worker_task_queue = str(uuid.uuid4())
+ original_patched = temporalio.workflow.patched
+
+ def pre_patch(patch_id: str) -> bool:
+ return False if patch_id == "recalc-activity-task-queue-2026-10" else original_patched(patch_id)
+
+ with patch.object(temporalio.workflow, "patched", side_effect=pre_patch):
+ run = await _run_with_activity_queue_worker(
+ activity_task_queue=activity_worker_task_queue,
+ activity_worker_task_queue=activity_worker_task_queue,
+ )
+
+ assert run.workflow_queue_calls == ["m1"]
+ assert run.activity_queue_calls == []
+ await temporalio.worker.Replayer(
+ workflows=[ExperimentMetricsRecalculationWorkflow],
+ workflow_runner=temporalio.worker.UnsandboxedWorkflowRunner(),
+ ).replay_workflow(run.history)