Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): route flagged organizations to the recalculation activity queue (#119618)

Rodrigo Ilorobc6c13fb179a
diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.py
index 923d35c25737..58b9e7ac7a41 100644
--- a/products/experiments/backend/recalculation.py
+++ b/products/experiments/backend/recalculation.py
@@ -57,6 +57,7 @@ _STALE_RECALC_THRESHOLD = timedelta(minutes=30)
# rate limiting manual reloads (including agents).
MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)
RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"
+RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG = "experiment-recalculation-activity-queue"
_RATE_LIMITED_TRIGGERS = frozenset(
{
ExperimentMetricsRecalculation.Trigger.MANUAL,
@@ -271,6 +272,7 @@ def start_metrics_recalculation_workflow(recalculation_id: str, *, team_id: int,
ExperimentMetricsRecalculationWorkflowInputs(
recalculation_id=recalculation_id,
fairness_key=organization_id,
+ activity_task_queue=_activity_task_queue(organization_id),
),
id=metrics_recalculation_workflow_id(recalculation_id),
task_queue=settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE,
@@ -309,15 +311,11 @@ def _cancel_superseded_workflows(recalculation_ids: list[str]) -> None:
pass
-def _refresh_window_enforced(experiment: Experiment) -> bool:
- # Local development skips the window so a developer can reload at will. Tests keep it, because they assert it.
- if settings.DEBUG and not settings.TEST:
- return False
- organization_id = str(experiment.team.organization_id)
- # Evaluated locally: this sits on the POST path, and a miss would otherwise block the request on a call
+def _organization_flag_enabled(flag_key: str, organization_id: str) -> bool:
+ # Evaluated locally: callers sit on request and start paths, and a miss would otherwise block them on a call
# to /flags. The flag has to target the organization id sent here, or it reads false for everyone.
return feature_enabled_or_false(
- RECALCULATION_RATE_LIMIT_FEATURE_FLAG,
+ flag_key,
organization_id,
groups={"organization": organization_id},
group_properties={"organization": {"id": organization_id}},
@@ -326,6 +324,23 @@ def _refresh_window_enforced(experiment: Experiment) -> bool:
)
+def _activity_task_queue(organization_id: str) -> str | None:
+ # A flag error keeps the run on the workflow's queue, because a failed start would roll the row back to FAILED.
+ try:
+ enabled = _organization_flag_enabled(RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG, organization_id)
+ except Exception as e:
+ capture_exception(e)
+ return None
+ return settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE if enabled else None
+
+
+def _refresh_window_enforced(experiment: Experiment) -> bool:
+ # Local development skips the window so a developer can reload at will. Tests keep it, because they assert it.
+ if settings.DEBUG and not settings.TEST:
+ return False
+ return _organization_flag_enabled(RECALCULATION_RATE_LIMIT_FEATURE_FLAG, str(experiment.team.organization_id))
+
+
def request_recalculation(experiment: Experiment, user: User | None, trigger: str = "manual") -> dict:
"""Create an idempotent batch recalculation request for all experiment metrics.
diff --git a/products/experiments/backend/test/test_metrics_recalculation_api.py b/products/experiments/backend/test/test_metrics_recalculation_api.py
index fb1103c6ebef..d1a3dfa87da6 100644
--- a/products/experiments/backend/test/test_metrics_recalculation_api.py
+++ b/products/experiments/backend/test/test_metrics_recalculation_api.py
@@ -17,6 +17,7 @@ from products.experiments.backend.models.experiment import (
ExperimentMetricResult,
ExperimentMetricsRecalculation,
)
+from products.experiments.backend.recalculation import RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG
from products.experiments.backend.temporal.models import ExperimentMetricsRecalculationWorkflowInputs
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -79,11 +80,36 @@ class TestMetricsRecalculationAPI(APIBaseTest):
# POST /metrics_recalculation/
# ------------------------------------------------------------------
+ @parameterized.expand(
+ [
+ # name, activity queue flag outcome, expected activity_task_queue
+ ("activity_queue_flag_off", False, None),
+ ("activity_queue_flag_on", True, settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE),
+ ("activity_queue_flag_error", RuntimeError("flags down"), None),
+ ]
+ )
@mock.patch("products.experiments.backend.recalculation.sync_connect")
@mock.patch("products.experiments.backend.recalculation.asyncio.run")
- def test_post_creates_and_starts_workflow(self, mock_run, mock_connect):
+ def test_post_creates_and_starts_workflow(
+ self,
+ _name: str,
+ flag_outcome: bool | Exception,
+ expected_activity_task_queue: str | None,
+ mock_run,
+ mock_connect,
+ ):
+ def evaluate_flag(key: str, *args, **kwargs) -> bool:
+ if key != RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG:
+ return False
+ if isinstance(flag_outcome, Exception):
+ raise flag_outcome
+ return flag_outcome
+
exp = self._launched_experiment()
- resp = self.client.post(self._post_url(exp.id), {"trigger": "manual"}, format="json")
+ with mock.patch(
+ "products.experiments.backend.recalculation.feature_enabled_or_false", side_effect=evaluate_flag
+ ):
+ resp = self.client.post(self._post_url(exp.id), {"trigger": "manual"}, format="json")
assert resp.status_code == status.HTTP_201_CREATED, resp.content
body = resp.json()
assert body["status"] == "pending"
@@ -92,7 +118,9 @@ class TestMetricsRecalculationAPI(APIBaseTest):
mock_connect.return_value.start_workflow.assert_called_once_with(
"experiment-metrics-recalculation-workflow",
ExperimentMetricsRecalculationWorkflowInputs(
- recalculation_id=body["id"], fairness_key=str(self.organization.id)
+ recalculation_id=body["id"],
+ fairness_key=str(self.organization.id),
+ activity_task_queue=expected_activity_task_queue,
),
id=f"experiment-metrics-recalculation-{body['id']}",
task_queue=settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE,