Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): keep queued recalc runs past the stale threshold (#128312)

Rodrigo Iloro68d33f0bca78
diff --git a/products/experiments/backend/recalculation.py b/products/experiments/backend/recalculation.py
index 2a29409a4049..bda522a17665 100644
--- a/products/experiments/backend/recalculation.py
+++ b/products/experiments/backend/recalculation.py
@@ -20,6 +20,8 @@ from django.utils import timezone
import posthoganalytics
from prometheus_client import Counter
from rest_framework.exceptions import Throttled, ValidationError
+from temporalio.client import WorkflowExecutionDescription, WorkflowExecutionStatus
+from temporalio.service import RPCError, RPCStatusCode
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.client.connection import ClickHouseUser, Workload
@@ -30,7 +32,7 @@ from posthog.models.scoping import get_current_team_id, team_scope
from posthog.models.user import User
from posthog.ph_client import feature_enabled_or_false
from posthog.settings import CLICKHOUSE_CLUSTER
-from posthog.temporal.common.client import sync_connect
+from posthog.temporal.common.client import async_connect, sync_connect
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window_end
from products.experiments.backend.metric_calculation.config import build_calculation_configs
@@ -58,6 +60,11 @@ from products.experiments.backend.temporal.models import (
# is the defense-in-depth backstop if that rollback itself fails.
_STALE_RECALC_THRESHOLD = timedelta(minutes=30)
+# A calc that waits on the activity or precompute queue has no time budget, so a row past _STALE_RECALC_THRESHOLD
+# stays active while Temporal says its workflow may still run, up to this ceiling. Past it, the row is stale
+# whatever Temporal says, so a stuck workflow or a Temporal outage cannot block an experiment for longer.
+_QUEUED_RECALC_CEILING = timedelta(hours=1)
+
# rate limiting manual reloads (including agents).
MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)
RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"
@@ -316,6 +323,51 @@ def start_metrics_recalculation_workflow(recalculation_id: str, *, team_id: int,
raise
+# Ceiling on the Temporal connect and describe for an old active row. The request holds the experiment row lock
+# while it waits.
+_WORKFLOW_STATUS_TIMEOUT_SECONDS = 3
+
+_CLOSED_WORKFLOW_STATUSES = frozenset(
+ {
+ WorkflowExecutionStatus.COMPLETED,
+ WorkflowExecutionStatus.FAILED,
+ WorkflowExecutionStatus.CANCELED,
+ WorkflowExecutionStatus.TERMINATED,
+ WorkflowExecutionStatus.CONTINUED_AS_NEW,
+ WorkflowExecutionStatus.TIMED_OUT,
+ }
+)
+
+
+async def _describe_recalculation_workflow(recalculation_id: str) -> WorkflowExecutionDescription:
+ temporal = await async_connect()
+ return await temporal.get_workflow_handle(metrics_recalculation_workflow_id(recalculation_id)).describe()
+
+
+def _workflow_may_still_run(recalculation_id: str) -> bool:
+ """Whether the workflow of an active row past _STALE_RECALC_THRESHOLD can still finish it.
+
+ A calc that waits on the activity or precompute queue has no time budget, so age alone does not prove that a
+ run is dead. Only a closed or missing workflow does. An unknown state keeps the row, because failing it would
+ cancel a run that may still be working.
+ """
+ try:
+ description = asyncio.run(
+ asyncio.wait_for(
+ _describe_recalculation_workflow(recalculation_id), timeout=_WORKFLOW_STATUS_TIMEOUT_SECONDS
+ )
+ )
+ except RPCError as e:
+ if e.status == RPCStatusCode.NOT_FOUND:
+ return False
+ capture_exception(e)
+ return True
+ except Exception as e:
+ capture_exception(e)
+ return True
+ return description.status not in _CLOSED_WORKFLOW_STATUSES
+
+
def _cancel_superseded_workflows(recalculation_ids: list[str]) -> None:
"""Best-effort cancellation of workflows whose rows were force-failed by the staleness cleanup."""
try:
@@ -479,9 +531,9 @@ def _request_recalculation(
# Activity-aware staleness: PENDING rows anchor on created_at (workflow never reached its start
# activity); IN_PROGRESS rows anchor on started_at (workflow began executing then stalled). A row past
- # the threshold is treated as dead and skipped, so a fresh recalc can start. Without this, a PENDING
- # row left orphaned by a Temporal-connect failure that also lost its rollback UPDATE would permanently
- # lock the experiment out of recalculations.
+ # the threshold is skipped here, and _queued_recalculation below decides whether it is still alive.
+ # Without this, a PENDING row left orphaned by a Temporal-connect failure that also lost its rollback
+ # UPDATE would permanently lock the experiment out of recalculations.
threshold = timezone.now() - _STALE_RECALC_THRESHOLD
existing = (
ExperimentMetricsRecalculation.objects.filter(experiment=experiment)
@@ -495,6 +547,11 @@ def _request_recalculation(
_recalculation_reuse_counter.inc()
return build_job_payload(existing, is_existing=True)
+ queued = _queued_recalculation(experiment)
+ if queued is not None:
+ _recalculation_reuse_counter.inc()
+ return build_job_payload(queued, is_existing=True)
+
if trigger in _RATE_LIMITED_TRIGGERS and _refresh_window_enforced(experiment):
# get the latest terminal run and check against the rate limiting rules
# if matched, increment counter and raise so the API answers 429 with Retry-After
@@ -565,8 +622,9 @@ def _request_recalculation(
def active_recalculation_filter() -> Q:
"""The rows that block a new recalculation: PENDING or IN_PROGRESS, inside the staleness bound.
- The scheduled discovery applies the same filter to many experiments in one query, so the two
- readers cannot disagree on which run is active.
+ The scheduled discovery applies the same filter to many experiments in one query. A row past the bound can
+ still be active while its workflow may run (_queued_recalculation); get_active_recalculation and the request
+ path check that one experiment at a time.
"""
threshold = timezone.now() - _STALE_RECALC_THRESHOLD
return Q(status=ExperimentMetricsRecalculation.Status.PENDING, created_at__gte=threshold) | Q(
@@ -576,12 +634,42 @@ def active_recalculation_filter() -> Q:
def get_active_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:
with team_scope(experiment.team_id, canonical=True):
- return (
+ active = (
ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment)
.filter(active_recalculation_filter())
.order_by("-created_at")
.first()
)
+ return active if active is not None else _queued_recalculation(experiment)
+
+
+def _queued_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:
+ """An active row between _STALE_RECALC_THRESHOLD and _QUEUED_RECALC_CEILING whose workflow may still run.
+
+ Callers hold the team scope. It asks Temporal only when such a row exists.
+ """
+ threshold = timezone.now() - _STALE_RECALC_THRESHOLD
+ ceiling = timezone.now() - _QUEUED_RECALC_CEILING
+ candidates = (
+ ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment)
+ .filter(
+ Q(
+ status=ExperimentMetricsRecalculation.Status.PENDING,
+ created_at__lt=threshold,
+ created_at__gte=ceiling,
+ )
+ | Q(
+ status=ExperimentMetricsRecalculation.Status.IN_PROGRESS,
+ started_at__lt=threshold,
+ started_at__gte=ceiling,
+ )
+ )
+ .order_by("-created_at")
+ )
+ for candidate in candidates:
+ if _workflow_may_still_run(str(candidate.id)):
+ return candidate
+ return None
def _terminal_recalculations(experiment: Experiment) -> QuerySet[ExperimentMetricsRecalculation]:
diff --git a/products/experiments/backend/test/test_recalculation_service.py b/products/experiments/backend/test/test_recalculation_service.py
index 5591dbbe3900..f5eca9e00819 100644
--- a/products/experiments/backend/test/test_recalculation_service.py
+++ b/products/experiments/backend/test/test_recalculation_service.py
@@ -1,16 +1,19 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
import pytest
import time_machine
from posthog.test.base import BaseTest, ClickhouseTestMixin
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
from django.test import override_settings
from django.utils import timezone
from parameterized import parameterized
from rest_framework.exceptions import ValidationError
+from temporalio.client import WorkflowExecutionStatus
+from temporalio.service import RPCError, RPCStatusCode
from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
@@ -58,6 +61,27 @@ def _calculation_key(experiment: Experiment, metric_uuid: str) -> str:
return calculation_config.calculation_key()
+_RECALCULATION_MODULE = "products.experiments.backend.recalculation"
+
+
+def _workflow_running() -> SimpleNamespace:
+ return SimpleNamespace(status=WorkflowExecutionStatus.RUNNING)
+
+
+def _workflow_closed() -> SimpleNamespace:
+ return SimpleNamespace(status=WorkflowExecutionStatus.COMPLETED)
+
+
+def _workflow_not_found() -> RPCError:
+ return RPCError("workflow not found", RPCStatusCode.NOT_FOUND, b"")
+
+
+def _describe(outcome: SimpleNamespace | Exception) -> AsyncMock:
+ if isinstance(outcome, Exception):
+ return AsyncMock(side_effect=outcome)
+ return AsyncMock(return_value=outcome)
+
+
@pytest.mark.django_db(transaction=True)
class TestRecalculationService(BaseTest):
def _flag(self, key: str) -> FeatureFlag:
@@ -284,6 +308,7 @@ class TestRecalculationService(BaseTest):
with (
patch("products.experiments.backend.recalculation.feature_enabled_or_false", side_effect=evaluate_flag),
patch("products.experiments.backend.recalculation.posthoganalytics.capture") as mock_capture,
+ patch(f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(_workflow_not_found())),
patch("products.experiments.backend.recalculation._cancel_superseded_workflows") as mock_cancel,
self.captureOnCommitCallbacks(execute=True),
):
@@ -335,16 +360,30 @@ class TestRecalculationService(BaseTest):
@parameterized.expand(
[
- # (name, status, stale_field) — both lifecycle anchors must release the experiment when their
- # respective timestamp is past the staleness threshold. Without this, a Temporal-connect failure
- # that also lost its rollback UPDATE permanently locks the experiment out of recalculations.
- ("pending_stale_by_created_at", "pending", "created_at"),
- ("in_progress_stale_by_started_at", "in_progress", "started_at"),
+ # (name, status, stale_field, age in minutes, workflow describe outcome, row kept). Both lifecycle
+ # anchors must release the experiment when their timestamp is past the staleness threshold and the
+ # workflow is gone, or a Temporal-connect failure that also lost its rollback UPDATE locks the
+ # experiment out of recalculations. A run whose workflow still runs, or whose state is unknown, keeps
+ # its row until the ceiling, because its calcs can wait in a queue with no time budget.
+ ("pending_stale_by_created_at", "pending", "created_at", 45, _workflow_not_found(), False),
+ ("in_progress_stale_by_started_at", "in_progress", "started_at", 45, _workflow_closed(), False),
+ ("pending_workflow_running", "pending", "created_at", 45, _workflow_running(), True),
+ ("in_progress_workflow_running", "in_progress", "started_at", 45, _workflow_running(), True),
+ ("in_progress_describe_error", "in_progress", "started_at", 45, TimeoutError(), True),
+ ("in_progress_past_ceiling", "in_progress", "started_at", 75, _workflow_running(), False),
]
)
- def test_request_recalculation_recovers_from_stale_active_row(self, name: str, status: str, stale_field: str):
+ def test_request_recalculation_recovers_from_stale_active_row(
+ self,
+ name: str,
+ status: str,
+ stale_field: str,
+ age_minutes: int,
+ describe_outcome: SimpleNamespace | Exception,
+ row_kept: bool,
+ ):
exp = self._launched_experiment(flag_key=f"stale-{name}")
- long_ago = timezone.now() - timedelta(hours=2)
+ long_ago = timezone.now() - timedelta(minutes=age_minutes)
stale_kwargs: dict = {"team": self.team, "experiment": exp, "status": status}
if stale_field == "created_at":
# created_at uses auto_now_add so we have to update after create.
@@ -354,18 +393,32 @@ class TestRecalculationService(BaseTest):
stale_row = ExperimentMetricsRecalculation.objects.create(**stale_kwargs, started_at=long_ago)
with (
+ patch(
+ f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(describe_outcome)
+ ) as mock_describe,
+ patch(f"{_RECALCULATION_MODULE}.capture_exception") as mock_capture_exception,
patch("products.experiments.backend.recalculation._cancel_superseded_workflows") as mock_cancel,
self.captureOnCommitCallbacks(execute=True),
):
result = request_recalculation(exp, self.user, "manual")
+ stale_row.refresh_from_db()
+ if row_kept:
+ assert result["is_existing"] is True
+ assert result["id"] == str(stale_row.id)
+ assert stale_row.status == status
+ mock_cancel.assert_not_called()
+ assert mock_capture_exception.called is isinstance(describe_outcome, TimeoutError)
+ return
+
# A fresh row was created (NOT the stale one), the stale row was marked FAILED, and its workflow
# was cancelled so it can't keep executing alongside the replacement run.
assert result["is_existing"] is False
assert result["id"] != str(stale_row.id)
- stale_row.refresh_from_db()
assert stale_row.status == "failed"
mock_cancel.assert_called_once_with([str(stale_row.id)])
+ # Past the ceiling the row is stale whatever Temporal says, so Temporal is not asked.
+ assert mock_describe.await_count == (0 if age_minutes > 60 else 1)
def test_request_recalculation_does_not_recover_recent_active_row(self):
# Defensive: a row created N minutes ago (well under the threshold) must still block, no matter the
@@ -436,6 +489,8 @@ class TestRecalculationService(BaseTest):
("fresh_in_progress", {"status": "in_progress", "started_at": "now"}, False, True, False),
("stale_pending", {"status": "pending"}, True, False, False),
("stale_in_progress", {"status": "in_progress", "started_at": "old"}, False, False, False),
+ # Past the staleness threshold but inside the ceiling, with a running workflow: still the active run.
+ ("queued_in_progress", {"status": "in_progress", "started_at": "queued"}, False, True, False),
("completed", {"status": "completed"}, False, False, True),
(
"failed_finalized_with_errors",
@@ -463,6 +518,8 @@ class TestRecalculationService(BaseTest):
kwargs["started_at"] = timezone.now()
elif kwargs.get("started_at") == "old":
kwargs["started_at"] = timezone.now() - timedelta(hours=2)
+ elif kwargs.get("started_at") == "queued":
+ kwargs["started_at"] = timezone.now() - timedelta(minutes=45)
if kwargs.get("completed_at") == "now":
kwargs["completed_at"] = timezone.now()
row = ExperimentMetricsRecalculation.objects.create(team=self.team, experiment=exp, **kwargs)
@@ -472,7 +529,8 @@ class TestRecalculationService(BaseTest):
created_at=timezone.now() - timedelta(hours=2)
)
- active = get_active_recalculation(exp)
+ with patch(f"{_RECALCULATION_MODULE}._describe_recalculation_workflow", new=_describe(_workflow_running())):
+ active = get_active_recalculation(exp)
latest = get_latest_recalculation(exp)
assert (active.id if active else None) == (row.id if row and expect_active else None)
assert (latest.id if latest else None) == (row.id if row and expect_latest else None)