Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

40.4 KB
Raw Blame
"""Service layer for experiment metrics recalculation.

Module-level free functions (not methods on ExperimentService) so the API view can compose them directly:

- ``request_recalculation`` — idempotent create: returns the active run if one exists, else creates a pending job.
- ``get_latest_recalculation`` — most recent recalc row for an experiment, or ``None``.
- ``get_run_results`` — read-back of per-metric results for a specific run, through ``MetricResultStore.for_run``
  (no FK on ExperimentMetricResult; the store finds a run's rows from the job row and the current metric config).
"""

import asyncio
from datetime import timedelta
from uuid import UUID

from django.conf import settings
from django.db import transaction
from django.db.models import Q, QuerySet
from django.utils import timezone

import posthoganalytics
from prometheus_client import Counter
from rest_framework.exceptions import Throttled, ValidationError
from temporalio.client import WorkflowExecutionDescription, WorkflowExecutionStatus
from temporalio.service import RPCError, RPCStatusCode

from posthog.clickhouse.client import sync_execute
from posthog.clickhouse.client.connection import ClickHouseUser, Workload
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.event_usage import groups
from posthog.exceptions_capture import capture_exception
from posthog.models.scoping import get_current_team_id, team_scope
from posthog.models.user import User
from posthog.ph_client import feature_enabled_or_false
from posthog.settings import CLICKHOUSE_CLUSTER
from posthog.temporal.common.client import async_connect, sync_connect

from products.experiments.backend.hogql_queries.base_query_utils import experiment_window_end
from products.experiments.backend.metric_calculation.config import build_calculation_configs
from products.experiments.backend.metric_calculation.results import MetricResultStore
from products.experiments.backend.metric_resolution import get_metrics_for_calculation
from products.experiments.backend.models.experiment import (
    Experiment,
    ExperimentMetricResult,
    ExperimentMetricsRecalculation,
)
from products.experiments.backend.result_serialization import strip_step_sessions
from products.experiments.backend.temporal.models import (
    METRICS_RECALCULATION_WORKFLOW_NAME,
    RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE,
    ExperimentMetricsRecalculationWorkflowInputs,
)

# How long an active (PENDING/IN_PROGRESS) row blocks new recalculations. Beyond this, the row is treated as
# stale and a fresh recalc is allowed. An IN_PROGRESS row counts from started_at, so it must outlast the phase
# after the start activity: the calc budget (METRIC_CALC_SCHEDULE_TO_CLOSE_TIMEOUT_SECONDS), the failure
# backstop and the finish step. test_in_progress_phase_ends_before_the_staleness_ttl keeps that sum below this
# value, so a legitimately-slow run can finish without being clobbered. It is also tight enough that an operator
# can recover within an hour if the workflow never started (Temporal connect failure, transient infra issue,
# etc.). See the rollback in start_metrics_recalculation_workflow for the happy-path failure handling; this TTL
# is the defense-in-depth backstop if that rollback itself fails.
_STALE_RECALC_THRESHOLD = timedelta(minutes=30)

# A calc that waits on the activity or precompute queue has no time budget, so a row past _STALE_RECALC_THRESHOLD
# stays active while Temporal says its workflow may still run, up to this ceiling. Past it, the row is stale
# whatever Temporal says, so a stuck workflow or a Temporal outage cannot block an experiment for longer.
_QUEUED_RECALC_CEILING = timedelta(hours=1)

# rate limiting manual reloads (including agents).
MIN_USER_RECALCULATION_INTERVAL = timedelta(minutes=5)
RECALCULATION_RATE_LIMIT_FEATURE_FLAG = "experiment-metric-recalculation-rate-limit"
RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG = "experiment-recalculation-activity-queue"
_RATE_LIMITED_TRIGGERS = frozenset(
    {
        ExperimentMetricsRecalculation.Trigger.MANUAL,
        ExperimentMetricsRecalculation.Trigger.AGENT_MCP,
    }
)

# A run scans every event from the start date to now, so the oldest experiments make the most expensive
# ClickHouse queries. request_recalculation records them as rejected runs and starts no workflow.
RECALCULATION_MAX_EXPERIMENT_AGE = timedelta(days=180)
RECALCULATION_AGE_LIMIT_FEATURE_FLAG = "experiment-recalculation-age-limit"
RECALCULATION_REJECTED_EVENT = "experiment recalculation rejected"
RECALCULATION_WINDOW_TOO_LONG_MESSAGE = (
    "This experiment started too long ago for its metrics to be recalculated. "
    "To measure new data, start a new experiment or move the start date closer to today."
)


class RecalculationRateLimited(Throttled):
    """A user-driven trigger landed inside MIN_USER_RECALCULATION_INTERVAL. The API layer renders it as a 429
    with a Retry-After header, so a client or an agent knows when the next run is allowed."""

    # DRF sets this in Throttled.__init__ and its handler turns it into Retry-After; the stubs omit it.
    wait: float | None

    default_code = "recalculation_rate_limited"
    # DRF appends "Expected available in N seconds." to this, so the wait is not repeated here.
    default_detail = "Metrics were recalculated less than 5 minutes ago."


# Mirrors STALE_RESULTS_THRESHOLD_HOURS in experimentMetricsLogic.ts. The frontend judges staleness with the
# browser clock, so this server-side check stops a client clock that runs ahead from recalculating on every visit.
STALE_REFRESH_MIN_RESULT_AGE = timedelta(hours=12)

# A daily timeseries point older than this no longer stands in for a recalculation on the cold-start read. The
# daily run happens once per day, so a fresh experiment always has a point inside the bound.
TIMESERIES_FALLBACK_MAX_AGE = timedelta(hours=24)

# `is_existing=True` reuse path — counts how often the idempotency guard saves us a workflow start.
# A sustained climb here without a matching climb in requests is the signal the frontend is double-posting.
_recalculation_reuse_counter = Counter(
    "experiment_metrics_recalculation_existing_run_reused",
    "POST requests that returned an existing active run instead of creating a new one (idempotent reuse).",
)
# Counts user-driven POSTs that landed inside MIN_USER_RECALCULATION_INTERVAL and got the latest run back.
_recalculation_rate_limited_counter = Counter(
    "experiment_metrics_recalculation_rate_limited",
    "POST requests answered with 429 because a user-driven run was requested inside the refresh window.",
)
# Fires whenever the 30-min staleness threshold marks a PENDING/IN_PROGRESS row FAILED so the experiment
# can recalculate again. A sustained climb is a leading indicator of Temporal connect failures or the
# rollback in start_metrics_recalculation_workflow itself failing.
_recalculation_stale_cleanup_counter = Counter(
    "experiment_metrics_recalculation_stale_rows_cleaned",
    "Stale recalc rows force-failed to release the per-experiment uniqueness constraint.",
)


def _derive_counters(recalc: ExperimentMetricsRecalculation, results: list[dict] | None = None) -> tuple[int, int]:
    """Counters are not stored on the row (PR1 contract): they're derived on read from result rows + errors.

    `completed_metrics` = ExperimentMetricResult rows with status=COMPLETED for this run's fingerprints.
    `failed_metrics`    = ExperimentMetricResult rows with status=FAILED + metric_errors keys that never made
                          it to a result row (discovery-step failures).

    Accepts an optional pre-computed `results` list to avoid recomputing fingerprints on the GET path where
    the same list is also surfaced to the client.

    Inherits the fingerprint-divergence hazard from `get_run_results`: `completed_metrics` can silently drop
    if settings that feed the calculation key change between the workflow's writes and this read.
    `failed_metrics` is partly immune because discovery-step failures come from `metric_errors` (stored on the
    row), not from result rows.
    """
    rows = results if results is not None else get_run_results(recalc)
    completed = sum(1 for r in rows if r["status"] == ExperimentMetricResult.Status.COMPLETED)
    failed_in_rows = sum(1 for r in rows if r["status"] == ExperimentMetricResult.Status.FAILED)
    uuids_with_row = {r["metric_uuid"] for r in rows}
    discovery_only_failures = sum(1 for uuid in (recalc.metric_errors or {}) if uuid not in uuids_with_row)
    return completed, failed_in_rows + discovery_only_failures


def get_live_query_progress(recalc: ExperimentMetricsRecalculation) -> dict | None:
    """Cumulative ClickHouse progress for an in-flight run: in-flight queries from system.processes plus
    queries already finished during the run from system.query_log, matched by query_id prefix.

    Returns None unless the run is IN_PROGRESS. No storage needed: each metric query is tagged with the
    deterministic client_query_id `experiment_metric_recalc_{recalc_id}_{metric_uuid}_attempt{attempt:02d}`,
    which ClickHouse stamps into query_id as `{team_id}_{client_query_id}_{random}`. The prefix matched below
    stops at the recalc id, so the metric and attempt suffixes do not affect it. system.processes only holds a query
    while it executes, and the metric queries are usually shorter than the poll interval, so processes
    alone reads zero for most of the run; the query_log branch keeps finished queries counted, making
    rows_read cumulative and roughly monotonic across the run (modulo query_log flush lag).

    `estimated_rows_total` is ClickHouse's own total_rows_approx for in-flight queries (revised upward
    mid-scan) plus the final read_rows of finished ones, so it can trail rows_read; treat rows_read as
    the primary signal. Temporal retries of a metric produce one query_log row per attempt and each attempt's
    rows are summed; that overcount is accepted for a decorative counter.
    """
    if recalc.status != ExperimentMetricsRecalculation.Status.IN_PROGRESS:
        return None

    # system.processes and system.query_log are cluster-global tables with no ClickHouse-side tenant scoping:
    # the team boundary is enforced only by the leading `{team_id}_` in the query_id prefix. Callers must pass
    # a request-scoped row (see get_recalculation_by_id); this is the defense-in-depth check that keeps a
    # future unscoped caller from summing another team's queries.
    if recalc.team_id != get_current_team_id():
        return None

    prefix = f"{recalc.team_id}_experiment_metric_recalc_{recalc.id}_%"
    # The prefix alone scopes rows to this run; the time bound exists so the query_log scan prunes to the
    # run's window instead of walking the whole table on every poll. The 60s pad absorbs clock skew between
    # the Django clock that stamped started_at and the ClickHouse clock behind event_time.
    since = (recalc.started_at or recalc.created_at) - timedelta(seconds=60)
    # timeout_overflow_mode must be 'throw'. Under a 'break' profile, a query that reaches max_execution_time
    # returns zero rows for this keyless aggregate instead of an error. The throw goes to the except below.
    try:
        with tags_context(product=Product.EXPERIMENTS, feature=Feature.CACHE_WARMUP, team_id=recalc.team_id):
            rows = sync_execute(
                """
                SELECT
                    sum(rows_read) AS rows_read,
                    sum(estimated_rows_total) AS estimated_rows_total
                FROM
                (
                    SELECT
                        read_rows AS rows_read,
                        total_rows_approx AS estimated_rows_total
                    FROM clusterAllReplicas(%(cluster)s, system.processes)
                    WHERE query_id LIKE %(prefix)s
                    UNION ALL
                    SELECT
                        read_rows AS rows_read,
                        read_rows AS estimated_rows_total
                    FROM clusterAllReplicas(%(cluster)s, system.query_log)
                    WHERE query_id LIKE %(prefix)s
                        AND type = 'QueryFinish'
                        AND event_date >= toDate(toDateTime(%(since)s))
                        AND event_time >= toDateTime(%(since)s)
                )
                SETTINGS skip_unavailable_shards=1, max_execution_time=2, timeout_overflow_mode='throw'
                """,
                {"cluster": CLICKHOUSE_CLUSTER, "prefix": prefix, "since": int(since.timestamp())},
                workload=Workload.ONLINE,
                team_id=recalc.team_id,
            )
    except Exception:
        # Best-effort, decorative read on the poll's hot path: a cluster hiccup here must never sink the core
        # recalculation payload (status + derived counters). Swallow, and let the run's live progress read null.
        capture_exception()
        return None

    rows_read, estimated_rows_total = rows[0]
    # All-zeros is a real, non-terminal state (queries not started yet, or finished but not flushed to
    # query_log), distinct from "run finished". Return the zeros so the poll can tell "in-flight, nothing
    # visible yet" from terminal null.
    return {
        "rows_read": int(rows_read or 0),
        "estimated_rows_total": int(estimated_rows_total or 0),
    }


def build_job_payload(
    recalc: ExperimentMetricsRecalculation,
    *,
    is_existing: bool | None = None,
    results: list[dict] | None = None,
    include_live_progress: bool = False,
) -> dict:
    """Shape a recalc row + derived counters as a dict the serializer can re-serialize.

    Returns model-native values (datetimes, ints, dicts) — DRF handles the wire format. The POST path passes
    `is_existing` to signal whether the workflow needs starting; the GET paths pass `results` so the same row
    list backs both the derived counters and the response's `results` field (no duplicate fingerprint work).

    `include_live_progress` opts the caller into one system.processes read per call (see get_live_query_progress);
    the poll (GET) path sets it so an in-flight run carries live ClickHouse progress, while the POST path does not
    (nothing is running yet).
    """
    completed_metrics, failed_metrics = _derive_counters(recalc, results=results)
    payload: dict = {
        "id": str(recalc.id),
        "experiment_id": recalc.experiment_id,
        "status": recalc.status,
        "total_metrics": recalc.total_metrics,
        "completed_metrics": completed_metrics,
        "failed_metrics": failed_metrics,
        "metric_errors": recalc.metric_errors,
        "metric_retries": recalc.metric_retries,
        "trigger": recalc.trigger,
        "created_at": recalc.created_at,
        "started_at": recalc.started_at,
        "completed_at": recalc.completed_at,
        "query_to": recalc.query_to,
    }
    if is_existing is not None:
        payload["is_existing"] = is_existing
    if include_live_progress:
        live_progress = get_live_query_progress(recalc)
        if live_progress is not None:
            payload.update(live_progress)
    return payload


def metrics_recalculation_workflow_id(recalculation_id: str) -> str:
    """Temporal workflow id of a recalculation run. The start, the cancel and the admin's Temporal link all
    find a running workflow by this id, so a format change orphans the workflows that are already running."""
    return f"experiment-metrics-recalculation-{recalculation_id}"


def cancel_recalculation_workflow(recalculation_id: str) -> None:
    """Best-effort cancel of a single recalc's Temporal workflow. Swallows failures (already-finished or
    never-started runs) so callers can pair it with a status write without the cancel masking that write."""
    _cancel_superseded_workflows([recalculation_id])


def start_metrics_recalculation_workflow(recalculation_id: str, *, team_id: int, organization_id: str) -> None:
    """Dispatch the recalculation Temporal workflow for a row that request_recalculation just created.

    Start every recalculation workflow through this function, so the workflow name and id, the task queues
    and the rollback stay in one place. If the start fails, the row is marked FAILED when it is safe to do so,
    and the exception propagates to the caller.
    """
    try:
        temporal = sync_connect()
        asyncio.run(
            temporal.start_workflow(
                METRICS_RECALCULATION_WORKFLOW_NAME,
                ExperimentMetricsRecalculationWorkflowInputs(
                    recalculation_id=recalculation_id,
                    activity_task_queue=_activity_task_queue(organization_id),
                ),
                id=metrics_recalculation_workflow_id(recalculation_id),
                task_queue=settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE,
            )
        )
    except Exception:
        # team-scoped filter: defense in depth so the rollback can never reach across teams even if
        # recalculation_id were ever sourced from somewhere less trusted than the row we just created.
        # start_workflow can raise after the server accepted the start (e.g. RPC deadline on the
        # response leg), so only roll back a row that is still PENDING with no query_to. A row past
        # mark_started belongs to its running workflow and proceeds untouched. In the narrow window
        # where only discovery ran, the rollback wins deliberately: the mark_started and
        # mark_completed guards then terminate that orphan cleanly, and the caller's retry (a new
        # POST, or the admin action again) starts the replacement.
        ExperimentMetricsRecalculation.objects.for_team(team_id).filter(
            id=recalculation_id,
            status=ExperimentMetricsRecalculation.Status.PENDING,
            query_to__isnull=True,
        ).update(status=ExperimentMetricsRecalculation.Status.FAILED)
        raise


# Ceiling on the Temporal connect and describe for an old active row. The request holds the experiment row lock
# while it waits.
_WORKFLOW_STATUS_TIMEOUT_SECONDS = 3

_CLOSED_WORKFLOW_STATUSES = frozenset(
    {
        WorkflowExecutionStatus.COMPLETED,
        WorkflowExecutionStatus.FAILED,
        WorkflowExecutionStatus.CANCELED,
        WorkflowExecutionStatus.TERMINATED,
        WorkflowExecutionStatus.CONTINUED_AS_NEW,
        WorkflowExecutionStatus.TIMED_OUT,
    }
)


async def _describe_recalculation_workflow(recalculation_id: str) -> WorkflowExecutionDescription:
    temporal = await async_connect()
    return await temporal.get_workflow_handle(metrics_recalculation_workflow_id(recalculation_id)).describe()


def _workflow_may_still_run(recalculation_id: str) -> bool:
    """Whether the workflow of an active row past _STALE_RECALC_THRESHOLD can still finish it.

    A calc that waits on the activity or precompute queue has no time budget, so age alone does not prove that a
    run is dead. Only a closed or missing workflow does. An unknown state keeps the row, because failing it would
    cancel a run that may still be working.
    """
    try:
        description = asyncio.run(
            asyncio.wait_for(
                _describe_recalculation_workflow(recalculation_id), timeout=_WORKFLOW_STATUS_TIMEOUT_SECONDS
            )
        )
    except RPCError as e:
        if e.status == RPCStatusCode.NOT_FOUND:
            return False
        capture_exception(e)
        return True
    except Exception as e:
        capture_exception(e)
        return True
    return description.status not in _CLOSED_WORKFLOW_STATUSES


def _cancel_superseded_workflows(recalculation_ids: list[str]) -> None:
    """Best-effort cancellation of workflows whose rows were force-failed by the staleness cleanup."""
    try:
        temporal = sync_connect()
    except Exception as e:
        capture_exception(e)
        return
    for recalculation_id in recalculation_ids:
        try:
            handle = temporal.get_workflow_handle(metrics_recalculation_workflow_id(recalculation_id))
            asyncio.run(handle.cancel())
        except Exception:
            # Expected for rows whose workflow never started (Temporal connect failure) or already finished.
            pass


def _organization_flag_enabled(flag_key: str, organization_id: str) -> bool:
    # Evaluated locally: callers sit on request and start paths, and a miss would otherwise block them on a call
    # to /flags. The flag has to target the organization id sent here, or it reads false for everyone.
    return feature_enabled_or_false(
        flag_key,
        organization_id,
        groups={"organization": organization_id},
        group_properties={"organization": {"id": organization_id}},
        only_evaluate_locally=True,
        send_feature_flag_events=False,
    )


def _activity_task_queue(organization_id: str) -> str | None:
    # A flag error keeps the run on the workflow's queue, because a failed start would roll the row back to FAILED.
    try:
        enabled = _organization_flag_enabled(RECALCULATION_ACTIVITY_QUEUE_FEATURE_FLAG, organization_id)
    except Exception as e:
        capture_exception(e)
        return None
    return settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE if enabled else None


def recalculation_clickhouse_user(task_queue: str) -> ClickHouseUser | None:
    # Only the slot-capped queues run as the experiments user; a queue that collapses into the main queue keeps the default.
    capped_task_queues = {
        settings.EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE,
        settings.EXPERIMENTS_RECALCULATION_PRECOMPUTE_TASK_QUEUE,
    } - {settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE}
    return ClickHouseUser.EXPERIMENTS if task_queue in capped_task_queues else None


def _refresh_window_enforced(experiment: Experiment) -> bool:
    # Local development skips the window so a developer can reload at will. Tests keep it, because they assert it.
    if settings.DEBUG and not settings.TEST:
        return False
    return _organization_flag_enabled(RECALCULATION_RATE_LIMIT_FEATURE_FLAG, str(experiment.team.organization_id))


def _recalculation_max_age(experiment: Experiment) -> timedelta | None:
    # A flag error turns the limit off, so a flag outage never rejects a recalculation.
    try:
        enabled = _organization_flag_enabled(RECALCULATION_AGE_LIMIT_FEATURE_FLAG, str(experiment.team.organization_id))
    except Exception as e:
        capture_exception(e)
        return None
    return RECALCULATION_MAX_EXPERIMENT_AGE if enabled else None


def _capture_recalculation_rejected(
    recalc: ExperimentMetricsRecalculation, user: User | None, experiment_age: timedelta, max_age: timedelta
) -> None:
    # Telemetry must never fail the request, so every error is swallowed.
    try:
        team = recalc.experiment.team
        posthoganalytics.capture(
            distinct_id=user.distinct_id if user and user.distinct_id else f"team_{team.id}",
            event=RECALCULATION_REJECTED_EVENT,
            properties={
                "experiment_id": recalc.experiment_id,
                "team_id": team.id,
                "organization_id": str(team.organization_id),
                "trigger": recalc.trigger,
                "recalculation_id": str(recalc.id),
                "experiment_age_days": experiment_age.days,
                "max_age_days": max_age.days,
            },
            groups=groups(organization=team.organization, team=team),
        )
    except Exception as e:
        capture_exception(e)


def _create_rejected_recalculation(
    experiment: Experiment, user: User | None, trigger: str, experiment_age: timedelta, max_age: timedelta
) -> ExperimentMetricsRecalculation:
    """A terminal FAILED run with the window error on every metric. Callers hold the team scope and the lock.

    completed_at makes the run the latest one, so a reload shows the error and no page-load heal runs again.
    The run takes query_to from the latest terminal run, so the results lookup of this run returns the rows
    of that run and the earlier results stay readable.
    """
    now = timezone.now()
    previous = _terminal_recalculations(experiment).order_by("-created_at").first()
    metrics = get_metrics_for_calculation(experiment)
    error = {
        "step": "request",
        "message": RECALCULATION_WINDOW_TOO_LONG_MESSAGE,
        "error_type": RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE,
        "retriable": False,
        "timestamp": now.isoformat(),
    }
    recalc = ExperimentMetricsRecalculation.objects.create(
        team=experiment.team,
        experiment=experiment,
        trigger=trigger,
        status=ExperimentMetricsRecalculation.Status.FAILED,
        created_by=user,
        total_metrics=len(metrics),
        metric_uuids=[m.uuid for m in metrics],
        metric_errors={m.uuid: dict(error) for m in metrics},
        query_to=previous.query_to if previous is not None else None,
        completed_at=now,
    )
    transaction.on_commit(lambda: _capture_recalculation_rejected(recalc, user, experiment_age, max_age))
    return recalc


def request_recalculation(experiment: Experiment, user: User | None, trigger: str = "manual") -> dict:
    """Create an idempotent batch recalculation request for all experiment metrics.

    If an active (pending or in_progress) run already exists for this experiment, returns the existing run's
    serialized payload with ``is_existing=True`` — the caller should NOT start a new workflow in that case.
    When the organization has the age-limit flag on, an experiment that started more than
    ``RECALCULATION_MAX_EXPERIMENT_AGE`` ago gets a rejected run: status FAILED, ``is_existing=False``, and the
    caller must not start a workflow for it either. A user-driven trigger inside
    ``MIN_USER_RECALCULATION_INTERVAL`` after the latest completed run finished raises
    ``RecalculationRateLimited`` when the organization has the rate-limit flag on. A ``stale_refresh`` whose
    latest terminal run has a ``query_to`` inside ``STALE_REFRESH_MIN_RESULT_AGE`` returns that run with
    ``is_existing=True``, and the caller must not start a workflow. Otherwise creates a fresh pending row.
    """
    return _request_recalculation(experiment, user, trigger, max_age=_recalculation_max_age(experiment))


def request_recalculation_without_age_limit(experiment: Experiment, user: User | None, trigger: str) -> dict:
    """``request_recalculation`` without the experiment age limit, so staff can still recalculate an old
    experiment from the admin panel."""
    return _request_recalculation(experiment, user, trigger, max_age=None)


def _request_recalculation(
    experiment: Experiment, user: User | None, trigger: str, *, max_age: timedelta | None
) -> dict:
    if not experiment.is_launched:
        raise ValidationError("Cannot recalculate metrics for experiment that hasn't started")

    with team_scope(experiment.team_id, canonical=True), transaction.atomic():
        # Serialize concurrent POSTs for this experiment by locking the Experiment row up front. Without this,
        # two simultaneous POSTs (double-click, retry storm, two tabs) both see no active recalc row and both
        # reach .create(); the second hits the unique_active_metrics_recalculation_per_experiment constraint
        # and returns HTTP 500. Locking the Experiment row queues the second POST behind the first, which
        # then sees the freshly-created row in its lookup and returns is_existing=True cleanly.
        Experiment.objects.select_for_update().filter(id=experiment.id).first()

        # Activity-aware staleness: PENDING rows anchor on created_at (workflow never reached its start
        # activity); IN_PROGRESS rows anchor on started_at (workflow began executing then stalled). A row past
        # the threshold is skipped here, and _queued_recalculation below decides whether it is still alive.
        # Without this, a PENDING row left orphaned by a Temporal-connect failure that also lost its rollback
        # UPDATE would permanently lock the experiment out of recalculations.
        threshold = timezone.now() - _STALE_RECALC_THRESHOLD
        existing = (
            ExperimentMetricsRecalculation.objects.filter(experiment=experiment)
            .filter(
                Q(status=ExperimentMetricsRecalculation.Status.PENDING, created_at__gte=threshold)
                | Q(status=ExperimentMetricsRecalculation.Status.IN_PROGRESS, started_at__gte=threshold)
            )
            .first()
        )
        if existing:
            _recalculation_reuse_counter.inc()
            return build_job_payload(existing, is_existing=True)

        queued = _queued_recalculation(experiment)
        if queued is not None:
            _recalculation_reuse_counter.inc()
            return build_job_payload(queued, is_existing=True)

        if trigger in _RATE_LIMITED_TRIGGERS and _refresh_window_enforced(experiment):
            # get the latest terminal run and check against the rate limiting rules
            # if matched, increment counter and raise so the API answers 429 with Retry-After
            latest = _terminal_recalculations(experiment).order_by("-created_at").first()
            if (
                latest is not None
                and latest.status == ExperimentMetricsRecalculation.Status.COMPLETED
                and latest.trigger != ExperimentMetricsRecalculation.Trigger.TIMESERIES_SYNC
                and latest.completed_at is not None
                and latest.completed_at >= timezone.now() - MIN_USER_RECALCULATION_INTERVAL
            ):
                _recalculation_rate_limited_counter.inc()
                next_allowed_at = latest.completed_at + MIN_USER_RECALCULATION_INTERVAL
                raise RecalculationRateLimited(wait=(next_allowed_at - timezone.now()).total_seconds())

        if trigger == ExperimentMetricsRecalculation.Trigger.STALE_REFRESH:
            latest = _terminal_recalculations(experiment).order_by("-created_at").first()
            if (
                latest is not None
                and latest.query_to is not None
                and latest.query_to > timezone.now() - STALE_REFRESH_MIN_RESULT_AGE
            ):
                return build_job_payload(latest, is_existing=True)

        # No fresh active row, but stale tombstones might still hold the per-experiment uniqueness constraint
        # (unique_active_metrics_recalculation_per_experiment). Mark them FAILED so the constraint releases
        # and the new row can land, and cancel their workflows after commit — a superseded run that is still
        # executing would otherwise keep burning worker slots and ClickHouse quota alongside its replacement.
        stale_ids = list(
            ExperimentMetricsRecalculation.objects.filter(
                experiment=experiment,
                status__in=[
                    ExperimentMetricsRecalculation.Status.PENDING,
                    ExperimentMetricsRecalculation.Status.IN_PROGRESS,
                ],
            ).values_list("id", flat=True)
        )
        if stale_ids:
            # No completed_at: that stamp is reserved for the workflow finalize step, and its absence is
            # what keeps superseded/tombstone rows out of get_latest_recalculation.
            ExperimentMetricsRecalculation.objects.filter(
                team=experiment.team, experiment=experiment, id__in=stale_ids
            ).update(status=ExperimentMetricsRecalculation.Status.FAILED)
            _recalculation_stale_cleanup_counter.inc(len(stale_ids))
            transaction.on_commit(lambda: _cancel_superseded_workflows([str(stale_id) for stale_id in stale_ids]))

        # After the stale cleanup, so a stale run of an old experiment is failed and its workflow cancelled too.
        experiment_age = timezone.now() - experiment.start_date if experiment.start_date is not None else None
        if max_age is not None and experiment_age is not None and experiment_age > max_age:
            rejected = _create_rejected_recalculation(experiment, user, trigger, experiment_age, max_age)
            return build_job_payload(rejected, is_existing=False)

        # Set total_metrics up front from the experiment definition so the client can show progress
        # ("N of M") immediately, before the workflow's discovery activity confirms the same count.
        metrics = get_metrics_for_calculation(experiment)
        recalc = ExperimentMetricsRecalculation.objects.create(
            team=experiment.team,
            experiment=experiment,
            trigger=trigger,
            status=ExperimentMetricsRecalculation.Status.PENDING,
            created_by=user,
            total_metrics=len(metrics),
            metric_uuids=[m.uuid for m in metrics],
        )
        return build_job_payload(recalc, is_existing=False)


def active_recalculation_filter() -> Q:
    """The rows that block a new recalculation: PENDING or IN_PROGRESS, inside the staleness bound.

    The scheduled discovery applies the same filter to many experiments in one query. A row past the bound can
    still be active while its workflow may run (_queued_recalculation); get_active_recalculation and the request
    path check that one experiment at a time.
    """
    threshold = timezone.now() - _STALE_RECALC_THRESHOLD
    return Q(status=ExperimentMetricsRecalculation.Status.PENDING, created_at__gte=threshold) | Q(
        status=ExperimentMetricsRecalculation.Status.IN_PROGRESS, started_at__gte=threshold
    )


def get_active_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:
    with team_scope(experiment.team_id, canonical=True):
        active = (
            ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment)
            .filter(active_recalculation_filter())
            .order_by("-created_at")
            .first()
        )
        return active if active is not None else _queued_recalculation(experiment)


def _queued_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:
    """An active row between _STALE_RECALC_THRESHOLD and _QUEUED_RECALC_CEILING whose workflow may still run.

    Callers hold the team scope. It asks Temporal only when such a row exists.
    """
    threshold = timezone.now() - _STALE_RECALC_THRESHOLD
    ceiling = timezone.now() - _QUEUED_RECALC_CEILING
    candidates = (
        ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment)
        .filter(
            Q(
                status=ExperimentMetricsRecalculation.Status.PENDING,
                created_at__lt=threshold,
                created_at__gte=ceiling,
            )
            | Q(
                status=ExperimentMetricsRecalculation.Status.IN_PROGRESS,
                started_at__lt=threshold,
                started_at__gte=ceiling,
            )
        )
        .order_by("-created_at")
    )
    for candidate in candidates:
        if _workflow_may_still_run(str(candidate.id)):
            return candidate
    return None


def _terminal_recalculations(experiment: Experiment) -> QuerySet[ExperimentMetricsRecalculation]:
    """Runs that really finished. Callers hold the team scope.

    completed_at is only ever stamped by the workflow's finalize step, so it separates runs that really
    finished from trigger-failure tombstones (status flipped to FAILED at create time, never started). Keying
    on metric_errors instead would hide a failed run whose failures live only in result rows.
    """
    return ExperimentMetricsRecalculation.objects.filter(team=experiment.team, experiment=experiment).filter(
        Q(status=ExperimentMetricsRecalculation.Status.COMPLETED)
        | (Q(status=ExperimentMetricsRecalculation.Status.FAILED) & Q(completed_at__isnull=False))
    )


def get_latest_recalculation(experiment: Experiment) -> ExperimentMetricsRecalculation | None:
    with team_scope(experiment.team_id, canonical=True):
        return _terminal_recalculations(experiment).order_by("-created_at").first()


def get_recalculation_by_id(experiment: Experiment, recalculation_id: str) -> ExperimentMetricsRecalculation | None:
    """Return the recalculation row for ``recalculation_id`` if it belongs to ``experiment``, else None.

    Enforces experiment scoping so the id-based GET cannot leak rows from a different experiment in the same team.
    Returns None for a malformed UUID rather than raising, so the calling view can answer with a clean 404.
    """
    try:
        uuid_value = UUID(recalculation_id)
    except (ValueError, TypeError):
        return None
    with team_scope(experiment.team_id, canonical=True):
        return ExperimentMetricsRecalculation.objects.filter(
            team=experiment.team, experiment=experiment, id=uuid_value
        ).first()


def get_run_results(recalc: ExperimentMetricsRecalculation) -> list[dict]:
    """Return the ExperimentMetricResult rows that belong to THIS run.

    Never returns rows from a previous run or from the timeseries workflow (which uses config fingerprints).

    Ended experiments retain completed rows with a compatible display key after project settings change, without
    making those rows reusable.
    """
    rows = MetricResultStore(experiment_id=recalc.experiment_id).for_run(recalc)
    return [
        {
            "metric_uuid": row.metric_uuid,
            "status": row.status,
            "result": strip_step_sessions(row.result),
            "error_message": row.error_message,
        }
        for row in rows
    ]


def build_timeseries_cold_start_payload(experiment: Experiment) -> dict | None:
    """Synthetic 'completed' recalculation payload built from each metric's latest completed timeseries point.

    Pure read. Used by GET /metrics_recalculation/latest as a cold-start placeholder when no real
    metrics-recalculation run exists yet. Timeseries rows live in ExperimentMetricResult under the metric's
    CONFIG fingerprint (not a per-run recalc fingerprint), so they're found without any recalc row.

    Only points younger than TIMESERIES_FALLBACK_MAX_AGE count, except that an ended experiment keeps its latest
    compatible point from any time since launch, including a result under earlier settings, when that point reaches
    the end date. The frontend
    accepts a fallback that covers every metric without starting a run, so an older point must read as a gap or
    the page would show weeks-old numbers as completed. Returns None when no metric has a fresh completed point
    (caller then keeps the 404). query_to and completed_at both pin to the freshest point's date. GET never triggers anything itself.
    """
    with team_scope(experiment.team_id, canonical=True):
        calculation_configs = build_calculation_configs(experiment)
        store = MetricResultStore(experiment_id=experiment.id)
        now = timezone.now()
        results: list[dict] = []
        latest_query_to = None
        for calculation_config in calculation_configs:
            # Bounded on both sides: the backfill writes end-of-day points, so today's point can carry a future
            # query_to that would surface here as a future completion time.
            row = None
            if experiment.is_stopped and experiment.start_date:
                row = store.latest_daily_point(
                    calculation_config, since=experiment.start_date, until=now, preserve_history=True
                )
                # A point whose data ends before the end date misses the final hours, so a run must compute them.
                # A backfill stamps the end of its day as query_to, but its data ends at completed_at.
                if row is not None:
                    data_until = min(row.query_to, row.completed_at or row.query_to)
                    if data_until < experiment_window_end(experiment, now):
                        row = None
            if row is None:
                row = store.latest_daily_point(calculation_config, since=now - TIMESERIES_FALLBACK_MAX_AGE, until=now)
            if row is None:
                continue
            results.append(
                {
                    "metric_uuid": row.metric_uuid,
                    "status": row.status,
                    "result": strip_step_sessions(row.result),
                    "error_message": None,
                }
            )
            if latest_query_to is None or row.query_to > latest_query_to:
                latest_query_to = row.query_to

        if not results:
            return None

        return {
            "id": "timeseries-fallback",
            "experiment_id": experiment.id,
            "status": ExperimentMetricsRecalculation.Status.COMPLETED,
            "total_metrics": len(calculation_configs),
            "completed_metrics": len(results),
            "failed_metrics": 0,
            "metric_errors": {},
            "metric_retries": {},
            "created_at": latest_query_to,
            "started_at": latest_query_to,
            "completed_at": latest_query_to,
            "query_to": latest_query_to,
            "results": results,
            "result_source": "timeseries_fallback",
        }