:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): keep queued recalc runs past the stale threshold (#128312)
Rodrigo Iloro68d33f0bca78
diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.pyindex 2a29409a4049..bda522a17665 100644--- a/products/experiments/backend/recalculation.py+++ b/products/experiments/backend/recalculation.py@@ -20,6 +20,8 @@ from django.utils import timezoneimport posthoganalyticsfrom prometheus_client import Counterfrom rest_framework.exceptions import Throttled, ValidationError+from temporalio.client import WorkflowExecutionDescription, WorkflowExecutionStatus+from temporalio.service import RPCError, RPCStatusCodefrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.client.connection import ClickHouseUser, Workload@@ -30,7 +32,7 @@ from posthog.models.scoping import get_current_team_id, team_scopefrom posthog.models.user import Userfrom posthog.ph_client import feature_enabled_or_falsefrom posthog.settings import CLICKHOUSE_CLUSTER-from posthog.temporal.common.client import sync_connect+from posthog.temporal.common.client import async_connect, sync_connectfrom products.experiments.backend.hogql_queries.base_query_utils import experiment_window_endfrom products.experiments.backend.metric_calculation.config import build_calculation_configs@@ -58,6 +60,11 @@ from products.experiments.backend.temporal.models import (# is the defense-in-depth backstop if that rollback itself fails._STALE_RECALC_THRESHOLD = timedelta(minutes=30)+# A calc that waits on the activity or precompute queue has no time budget, so a row past _STALE_RECALC_THRESHOLD+# stays active while Temporal says its workflow may still run, up to this ceiling. Past it, the row is stale+# whatever Temporal says, so a stuck workflow or a Temporal outage cannot block an experiment for longer.+_QUEUED_RECALC_CEILING = timedelta(hours=1)+# rate limiting manual reloads (including agents).MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"@@ -316,6 +323,51 @@ def start_metrics_recalculation_workflow(recalculation_id: str, *, team_id: int,raise+# Ceiling on the Temporal connect and describe for an old active row. The request holds the experiment row lock+# while it waits.+_WORKFLOW_STATUS_TIMEOUT_SECONDS = 3++_CLOSED_WORKFLOW_STATUSES = frozenset(+ {+ WorkflowExecutionStatus.COMPLETED,+ WorkflowExecutionStatus.FAILED,+ WorkflowExecutionStatus.CANCELED,+ WorkflowExecutionStatus.TERMINATED,+ WorkflowExecutionStatus.CONTINUED_AS_NEW,+ WorkflowExecutionStatus.TIMED_OUT,+ }+)+++async def _describe_recalculation_workflow(recalculation_id: str) -> WorkflowExecutionDescription:+ temporal = await async_connect()+ return await temporal.get_workflow_handle(metrics_recalculation_workflow_id(recalculation_id)).describe()+++def _workflow_may_still_run(recalculation_id: str) -> bool:+ """Whether the workflow of an active row past _STALE_RECALC_THRESHOLD can still finish it.++ A calc that waits on the activity or precompute queue has no time budget, so age alone does not prove that a+ run is dead. Only a closed or missing workflow does. An unknown state keeps the row, because failing it would+ cancel a run that may still be working.+ """+ try:+ description = asyncio.run(+ asyncio.wait_for(+ _describe_recalculation_workflow(recalculation_id), timeout=_WORKFLOW_STATUS_TIMEOUT_SECONDS+ )+ )+ except RPCError as e:+ if e.status == RPCStatusCode.NOT_FOUND:+ return False+ capture_exception(e)+ return True+ except Exception as e:+ capture_exception(e)+ return True+ return description.status not in _CLOSED_WORKFLOW_STATUSES++def _cancel_superseded_workflows(recalculation_ids: list[str]) -> None:"""Best-effort cancellation of workflows whose rows were force-failed by the staleness cleanup."""try:@@ -479,9 +531,9 @@ def _request_recalculation(# Activity-aware staleness: PENDING rows anchor on created_at (workflow never reached its start# activity); IN_PROGRESS rows anchor on started_at (workflow began executing then stalled). A row past- # the threshold is treated as dead and skipped, so a fresh recalc can start. Without this, a PENDING- # row left orphaned by a Temporal-connect failure that also lost its rollback UPDATE would permanently- # lock the experiment out of recalculations.+ # the threshold is skipped here, and _queued_recalculation below decides whether it is still alive.+ # Without this, a PENDING row left orphaned by a Temporal-connect failure that also lost its rollback+ # UPDATE would permanently lock the experiment out of recalculations.threshold = timezone.now() - _STALE_RECALC_THRESHOLDexisting = (ExperimentMetricsRecalculation.objects.filter(experiment=experiment)@@ -495,6 +547,11 @@ def _request_recalculation(_recalculation_reuse_counter.inc()return build_job_payload(existing, is_existing=True)+ queued = _queued_recalculation(experiment)+ if queued is not None:+ _recalculation_reuse_counter.inc()+ return build_job_payload(queued, is_existing=True)+if trigger in _RATE_LIMITED_TRIGGERS and _refresh_window_enforced(experiment):# get the latest terminal run and check against the rate limiting rules# if matched, increment counter and raise so the API answers 429 with Retry-After@@ -565,8 +622,9 @@ def _request_recalculation(def active_recalculation_filter() -> Q:"""The rows that block a new recalculation: PENDING or IN_PROGRESS, inside the staleness bound.- The scheduled discovery applies the same filter to many experiments in one query, so the two- readers cannot disagree on which run is active.+ The scheduled discovery applies the same filter to many experiments in one query. A row past the bound can+ still be active while its workflow may run (_queued_recalculation); get_active_recalculation and the request+ path check that one experiment at a time."""threshold = timezone.now() - _STALE_RECALC_THRESHOLDreturn Q(status=ExperimentMetricsRecalculation.Status.PENDING, created_at__gte=threshold) | Q(@@ -576,12 +634,42 @@ def active_recalculation_filter() -> Q:def get_active_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:with team_scope(experiment.team_id, canonical=True):- return (+ active = (ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment).filter(active_recalculation_filter()).order_by("-created_at").first())+ return active if active is not None else _queued_recalculation(experiment)+++def _queued_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:+ """An active row between _STALE_RECALC_THRESHOLD and _QUEUED_RECALC_CEILING whose workflow may still run.++ Callers hold the team scope. It asks Temporal only when such a row exists.+ """+ threshold = timezone.now() - _STALE_RECALC_THRESHOLD+ ceiling = timezone.now() - _QUEUED_RECALC_CEILING+ candidates = (+ ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment)+ .filter(+ Q(+ status=ExperimentMetricsRecalculation.Status.PENDING,+ created_at__lt=threshold,+ created_at__gte=ceiling,+ )+ | Q(+ status=ExperimentMetricsRecalculation.Status.IN_PROGRESS,+ started_at__lt=threshold,+ started_at__gte=ceiling,+ )+ )+ .order_by("-created_at")+ )+ for candidate in candidates:+ if _workflow_may_still_run(str(candidate.id)):+ return candidate+ return Nonedef _terminal_recalculations(experiment: Experiment) -> QuerySet[ExperimentMetricsRecalculation]:diff --git a/products/experiments/backend/test/test_recalculation_service.py b/products/experiments/backend/test/test_recalculation_service.pyindex 5591dbbe3900..f5eca9e00819 100644--- a/products/experiments/backend/test/test_recalculation_service.py+++ b/products/experiments/backend/test/test_recalculation_service.py@@ -1,16 +1,19 @@from collections.abc import Callablefrom datetime import UTC, datetime, timedelta+from types import SimpleNamespaceimport pytestimport time_machinefrom posthog.test.base import BaseTest, ClickhouseTestMixin-from unittest.mock import patch+from unittest.mock import AsyncMock, patchfrom django.test import override_settingsfrom django.utils import timezonefrom parameterized import parameterizedfrom rest_framework.exceptions import ValidationError+from temporalio.client import WorkflowExecutionStatus+from temporalio.service import RPCError, RPCStatusCodefrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.query_tagging import Feature, Product, tags_context@@ -58,6 +61,27 @@ def _calculation_key(experiment: Experiment, metric_uuid: str) -> str:return calculation_config.calculation_key()+_RECALCULATION_MODULE = "products.experiments.backend.recalculation"+++def _workflow_running() -> SimpleNamespace:+ return SimpleNamespace(status=WorkflowExecutionStatus.RUNNING)+++def _workflow_closed() -> SimpleNamespace:+ return SimpleNamespace(status=WorkflowExecutionStatus.COMPLETED)+++def _workflow_not_found() -> RPCError:+ return RPCError("workflow not found", RPCStatusCode.NOT_FOUND, b"")+++def _describe(outcome: SimpleNamespace | Exception) -> AsyncMock:+ if isinstance(outcome, Exception):+ return AsyncMock(side_effect=outcome)+ return AsyncMock(return_value=outcome)++@pytest.mark.django_db(transaction=True)class TestRecalculationService(BaseTest):def _flag(self, key: str) -> FeatureFlag:@@ -284,6 +308,7 @@ class TestRecalculationService(BaseTest):with (patch("products.experiments.backend.recalculation.feature_enabled_or_false", side_effect=evaluate_flag),patch("products.experiments.backend.recalculation.posthoganalytics.capture") as mock_capture,+ patch(f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(_workflow_not_found())),patch("products.experiments.backend.recalculation._cancel_superseded_workflows") as mock_cancel,self.captureOnCommitCallbacks(execute=True),):@@ -335,16 +360,30 @@ class TestRecalculationService(BaseTest):@parameterized.expand([- # (name, status, stale_field) — both lifecycle anchors must release the experiment when their- # respective timestamp is past the staleness threshold. Without this, a Temporal-connect failure- # that also lost its rollback UPDATE permanently locks the experiment out of recalculations.- ("pending_stale_by_created_at", "pending", "created_at"),- ("in_progress_stale_by_started_at", "in_progress", "started_at"),+ # (name, status, stale_field, age in minutes, workflow describe outcome, row kept). Both lifecycle+ # anchors must release the experiment when their timestamp is past the staleness threshold and the+ # workflow is gone, or a Temporal-connect failure that also lost its rollback UPDATE locks the+ # experiment out of recalculations. A run whose workflow still runs, or whose state is unknown, keeps+ # its row until the ceiling, because its calcs can wait in a queue with no time budget.+ ("pending_stale_by_created_at", "pending", "created_at", 45, _workflow_not_found(), False),+ ("in_progress_stale_by_started_at", "in_progress", "started_at", 45, _workflow_closed(), False),+ ("pending_workflow_running", "pending", "created_at", 45, _workflow_running(), True),+ ("in_progress_workflow_running", "in_progress", "started_at", 45, _workflow_running(), True),+ ("in_progress_describe_error", "in_progress", "started_at", 45, TimeoutError(), True),+ ("in_progress_past_ceiling", "in_progress", "started_at", 75, _workflow_running(), False),])- def test_request_recalculation_recovers_from_stale_active_row(self, name: str, status: str, stale_field: str):+ def test_request_recalculation_recovers_from_stale_active_row(+ self,+ name: str,+ status: str,+ stale_field: str,+ age_minutes: int,+ describe_outcome: SimpleNamespace | Exception,+ row_kept: bool,+ ):exp = self._launched_experiment(flag_key=f"stale-{name}")- long_ago = timezone.now() - timedelta(hours=2)+ long_ago = timezone.now() - timedelta(minutes=age_minutes)stale_kwargs: dict = {"team": self.team, "experiment": exp, "status": status}if stale_field == "created_at":# created_at uses auto_now_add so we have to update after create.@@ -354,18 +393,32 @@ class TestRecalculationService(BaseTest):stale_row = ExperimentMetricsRecalculation.objects.create(**stale_kwargs, started_at=long_ago)with (+ patch(+ f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(describe_outcome)+ ) as mock_describe,+ patch(f"{_RECALCULATION_MODULE}.capture_exception") as mock_capture_exception,patch("products.experiments.backend.recalculation._cancel_superseded_workflows") as mock_cancel,self.captureOnCommitCallbacks(execute=True),):result = request_recalculation(exp, self.user, "manual")+ stale_row.refresh_from_db()+ if row_kept:+ assert result["is_existing"] is True+ assert result["id"] == str(stale_row.id)+ assert stale_row.status == status+ mock_cancel.assert_not_called()+ assert mock_capture_exception.called is isinstance(describe_outcome, TimeoutError)+ return+# A fresh row was created (NOT the stale one), the stale row was marked FAILED, and its workflow# was cancelled so it can't keep executing alongside the replacement run.assert result["is_existing"] is Falseassert result["id"] != str(stale_row.id)- stale_row.refresh_from_db()assert stale_row.status == "failed"mock_cancel.assert_called_once_with([str(stale_row.id)])+ # Past the ceiling the row is stale whatever Temporal says, so Temporal is not asked.+ assert mock_describe.await_count == (0 if age_minutes > 60 else 1)def test_request_recalculation_does_not_recover_recent_active_row(self):# Defensive: a row created N minutes ago (well under the threshold) must still block, no matter the@@ -436,6 +489,8 @@ class TestRecalculationService(BaseTest):("fresh_in_progress", {"status": "in_progress", "started_at": "now"}, False, True, False),("stale_pending", {"status": "pending"}, True, False, False),("stale_in_progress", {"status": "in_progress", "started_at": "old"}, False, False, False),+ # Past the staleness threshold but inside the ceiling, with a running workflow: still the active run.+ ("queued_in_progress", {"status": "in_progress", "started_at": "queued"}, False, True, False),("completed", {"status": "completed"}, False, False, True),("failed_finalized_with_errors",@@ -463,6 +518,8 @@ class TestRecalculationService(BaseTest):kwargs["started_at"] = timezone.now()elif kwargs.get("started_at") == "old":kwargs["started_at"] = timezone.now() - timedelta(hours=2)+ elif kwargs.get("started_at") == "queued":+ kwargs["started_at"] = timezone.now() - timedelta(minutes=45)if kwargs.get("completed_at") == "now":kwargs["completed_at"] = timezone.now()row = ExperimentMetricsRecalculation.objects.create(team=self.team, experiment=exp, **kwargs)@@ -472,7 +529,8 @@ class TestRecalculationService(BaseTest):created_at=timezone.now() - timedelta(hours=2))- active = get_active_recalculation(exp)+ with patch(f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(_workflow_running())):+ active = get_active_recalculation(exp)latest = get_latest_recalculation(exp)assert (active.id if active else None) == (row.id if row and expect_active else None)assert (latest.id if latest else None) == (row.id if row and expect_latest else None)