:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { MakeLogicType, actions, afterMount, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea'
import posthog from 'posthog-js'
import { lemonToast } from '@posthog/lemon-ui'
import { SetupTaskId, globalSetupLogic } from 'lib/components/ProductSetup'
import { dayjs } from 'lib/dayjs'
import { preflightLogic } from 'lib/logic/preflightLogic'
import { projectLogic } from 'scenes/projectLogic'
import type { CachedNewExperimentQueryResponse, ExperimentMetric } from '~/queries/schema/schema-general'
import { Experiment, ExperimentStatus } from '~/types'
import type { ExperimentIdType } from '~/types'
import { getExperimentStatus, isLaunched } from 'products/experiments/frontend/experimentStatus'
import {
experimentsMetricsRecalculationCreate,
experimentsMetricsRecalculationLatestRetrieve,
experimentsMetricsRecalculationRetrieve,
} from 'products/experiments/frontend/generated/api'
import type {
ExperimentMetricsRecalculationJobApi,
ExperimentMetricsRecalculationLatestApi,
ExperimentMetricsRecalculationRequestTriggerEnumApi,
ExperimentMetricsRecalculationRunApi,
ResultSourceEnumApi,
} from 'products/experiments/frontend/generated/api.schemas'
import { type ExperimentSavedMetric, sharedMetricsToExperimentMetrics } from './utils'
/**
* Why a load stopped waiting and showed a retryable error instead. Each one emits an `abandoned` event, the
* only signal a hung load leaves.
*/
export type RecalculationAbandonReason = 'latest_timeout' | 'poll_timeout' | 'poll_failed' | 'poll_duration_cap'
function reportExperimentMetricRecalculation(
status: 'triggered' | 'completed' | 'failed' | 'abandoned',
properties: {
experiment_id: number
recalculation_id: string | null
trigger?: ExperimentMetricsRecalculationRequestTriggerEnumApi
is_existing?: boolean
total_metrics?: number
succeeded?: number
failed?: number
duration_ms?: number
poll_count?: number
reason?: RecalculationAbandonReason
}
): void {
posthog.capture('experiment metric recalculation', { status, ...properties })
}
/**
* The shape every recalculation endpoint returns. POST answers with the queued job plus is_existing; the by-id
* poll and latest add results, retry state and live progress, so those fields are optional here.
*/
export type RecalculationPayload = Omit<ExperimentMetricsRecalculationJobApi, 'is_existing'> &
Partial<
Pick<ExperimentMetricsRecalculationRunApi, 'metric_retries' | 'results' | 'rows_read' | 'estimated_rows_total'>
> &
Partial<Pick<ExperimentMetricsRecalculationLatestApi, 'result_source'>>
/**
* This logic can only handle state when an experiment is present.
*/
export interface ExperimentMetricsLogicProps {
experiment: Experiment
}
// A manual reload inside this window after the latest completed run finished is blocked, the same five
// minutes a dashboard waits between bulk refreshes. The backend applies the same window and returns that run.
const MIN_MANUAL_REFRESH_INTERVAL_MINUTES = 5
const RECALCULATION_POLL_INTERVAL_MS = 2000
/** A run past the backend's 30-minute staleness threshold is usually waiting for a slot in a capped queue. */
const QUEUED_RUN_AGE_MS = 30 * 60 * 1000
const QUEUED_RUN_POLL_INTERVAL_MS = 60 * 1000
const MAX_POLL_RETRIES = 5
/**
* Mirrors the backend's 1-hour ceiling (_QUEUED_RECALC_CEILING): a run still non-terminal past it is treated
* as dead, and the backend fails it on the next trigger. Without this cap every tab that resumed the run would
* poll its id (and the per-tick live-progress query) forever.
*/
const MAX_POLL_DURATION_MS = 60 * 60 * 1000
// A request that never settles neither resolves nor rejects, so without a deadline the page loads forever.
const LATEST_REQUEST_TIMEOUT_MS = 30 * 1000
const POLL_REQUEST_TIMEOUT_MS = 15 * 1000
/** Each stale refresh is a full ClickHouse recalculation, so this threshold sets the load that page visits add. */
const STALE_RESULTS_THRESHOLD_HOURS = 12
/** A paused experiment collects no new data. An exposure-frozen one still collects metric events. */
const STALE_REFRESH_STATUSES: ReadonlySet<ExperimentStatus> = new Set([
ExperimentStatus.Running,
ExperimentStatus.ExposureFrozen,
])
function pollIntervalMs(runAgeMs: number): number {
return runAgeMs > QUEUED_RUN_AGE_MS ? QUEUED_RUN_POLL_INTERVAL_MS : RECALCULATION_POLL_INTERVAL_MS
}
class RecalculationRequestTimeoutError extends Error {}
/**
* Rejects with RecalculationRequestTimeoutError after `timeoutMs` and aborts the request. The race settles the
* call even if the transport ignores the abort signal.
*/
async function withRequestTimeout<T>(request: (signal: AbortSignal) => Promise<T>, timeoutMs: number): Promise<T> {
const controller = new AbortController()
let timerId: ReturnType<typeof setTimeout> | undefined
const deadline = new Promise<never>((_, reject) => {
timerId = setTimeout(() => {
reject(new RecalculationRequestTimeoutError(`Request timed out after ${timeoutMs}ms`))
controller.abort()
}, timeoutMs)
})
try {
return await Promise.race([request(controller.signal), deadline])
} finally {
clearTimeout(timerId)
}
}
const SLOW_LOAD_MESSAGE = 'Loading results is taking longer than expected. Refresh the results to try again.'
const ABANDON_MESSAGES: Record<RecalculationAbandonReason, string> = {
latest_timeout: SLOW_LOAD_MESSAGE,
poll_timeout: SLOW_LOAD_MESSAGE,
poll_duration_cap: SLOW_LOAD_MESSAGE,
poll_failed: "Couldn't load the recalculation results. Refresh the results to try again.",
}
export const RECALCULATION_STATUSES = {
pending: 'pending',
in_progress: 'in_progress',
completed: 'completed',
failed: 'failed',
} as const
/** pinned: the backend writes this error_type on a run it rejects because the experiment is too old. */
const RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE = 'recalculation_window_too_long'
/** Transient per-metric retry state written by the calc activity between failed attempts. */
export interface MetricRetryInfo {
attempt: number
max_attempts: number
error_type: string
/** The server error that triggered the retry, surfaced so the UI can show why. */
message?: string
next_retry_at?: string
}
/**
* One metric type's metrics (inline + shared) in the order results are positionally mapped against.
*/
const metricsInOrder = (experiment: Experiment, type: 'primary' | 'secondary'): ExperimentMetric[] => {
const sharedMetrics = sharedMetricsToExperimentMetrics(experiment.saved_metrics as ExperimentSavedMetric[], type)
const inline = (type === 'primary' ? experiment.metrics : experiment.metrics_secondary) || []
return [...(inline as ExperimentMetric[]), ...sharedMetrics]
}
/**
* Every metric uuid on the experiment now, across primary and secondary (inline + shared). Metrics without
* a uuid are skipped: they can't be matched against a run's per-uuid results, so they never signal a gap.
*/
const currentMetricUuids = (experiment: Experiment): string[] =>
[
...metricsInOrder(experiment, 'primary').map((metric) => metric.uuid),
...metricsInOrder(experiment, 'secondary').map((metric) => metric.uuid),
].filter((uuid): uuid is string => !!uuid)
/** One entry of `metric_errors`: a metric's terminal failure, as the backend records it. */
type MetricErrorEntry = { step?: string; message?: string; error_type?: string; retriable?: boolean }
const metricErrorEntries = (recalculation: RecalculationPayload): Record<string, MetricErrorEntry> =>
(recalculation.metric_errors as Record<string, MetricErrorEntry> | null) ?? {}
const rejectionMessage = (recalculation: RecalculationPayload): string | null =>
Object.values(metricErrorEntries(recalculation)).find(
(entry) => entry.error_type === RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE
)?.message ?? null
/**
* Metric uuids the run has an entry for: a result or an error. A failure, retriable or not, counts as
* covered, because only a manual retry or a manual recalculation runs a failed metric again.
*/
const coveredMetricUuids = (recalculation: RecalculationPayload): string[] => [
...(recalculation.results ?? []).map(({ metric_uuid }) => metric_uuid),
...Object.keys(metricErrorEntries(recalculation)),
]
/**
* The trigger that fills a gap, by where the latest payload came from. A real run is healed in place:
* heal_latest_run reuses its window, so metrics with rows load from cache. The timeseries fallback is
* not a run, so there is no window to reuse and nothing to dim; a cold_run starts fresh.
*/
const HEAL_TRIGGER_BY_RESULT_SOURCE = {
recalculation: 'heal_latest_run',
timeseries_fallback: 'cold_run',
} as const satisfies Record<ResultSourceEnumApi, ExperimentMetricsRecalculationRequestTriggerEnumApi>
const recalculationHasGap = (experiment: Experiment, recalculation: RecalculationPayload): boolean => {
if (
recalculation.status === RECALCULATION_STATUSES.pending ||
recalculation.status === RECALCULATION_STATUSES.in_progress
) {
return false
}
const coveredUuids = new Set(coveredMetricUuids(recalculation))
return currentMetricUuids(experiment).some((uuid) => !coveredUuids.has(uuid))
}
const resultsAreStale = (experiment: Experiment, recalculation: RecalculationPayload): boolean =>
STALE_REFRESH_STATUSES.has(getExperimentStatus(experiment)) &&
currentMetricUuids(experiment).length > 0 &&
!!recalculation.query_to &&
dayjs.utc().diff(dayjs.utc(recalculation.query_to), 'hour', true) > STALE_RESULTS_THRESHOLD_HOURS
type MetricErrorState = { detail: string; retriable: boolean } | null
type ResolveByUuid<T> = (uuid: string) => T
/**
* Metric uuids that currently show something, a result OR an error, across primary and secondary. A
* non-cold recalculation marks these as recalculating: the stale value (or stale error) stays on screen and
* the metric header shows a loading tag until the fresh one lands. Errored metrics are included so a
* retry shows the tag too.
*/
const metricUuidsToMarkRecalculating = (
experiment: Experiment,
primaryResults: readonly (CachedNewExperimentQueryResponse | undefined)[],
secondaryResults: readonly (CachedNewExperimentQueryResponse | undefined)[],
primaryErrors: readonly (unknown | null)[],
secondaryErrors: readonly (unknown | null)[]
): string[] => [
...metricsInOrder(experiment, 'primary')
.map((metric) => metric.uuid as string)
.filter((_, index) => primaryResults[index] !== undefined || !!primaryErrors[index]),
...metricsInOrder(experiment, 'secondary')
.map((metric) => metric.uuid as string)
.filter((_, index) => secondaryResults[index] !== undefined || !!secondaryErrors[index]),
]
/**
* One value per metric, in `metricsInOrder` order. Curried: bind (experiment, type) once, then feed a
* per-uuid resolver, the only thing that differs between results and errors.
*/
const alignByMetricPosition =
(experiment: Experiment, type: 'primary' | 'secondary') =>
<T>(resolve: ResolveByUuid<T>): T[] =>
metricsInOrder(experiment, type).map((metric) => resolve(metric.uuid as string))
/**
* Resolver: a polled metric's computed result, or undefined if the run hasn't produced one yet.
*/
const resolveResultByUuid = (
polledResults: readonly { metric_uuid: string; result: unknown }[] | undefined
): ResolveByUuid<CachedNewExperimentQueryResponse> => {
const resultByUuid = new Map((polledResults ?? []).map((r) => [r.metric_uuid, r.result]))
return (uuid) => resultByUuid.get(uuid) as CachedNewExperimentQueryResponse
}
/**
* Resolver: a metric's failure as `{ detail, retriable }` (the MetricErrorState shape), or null. `metric_errors`
* wins (it covers FAILED rows AND discovery-step failures absent from `results`), falling back to a
* failed row's error_message. A rejected run carries the earlier run's rows, so its error is dropped on a
* metric that has a row: the banner shows the message and the cell keeps the earlier result.
* `_shown_failure` in products/experiments/backend/health/latest_calculation.py repeats this rule for the health
* checks, so change both together.
*/
const resolveErrorByUuid = (recalculation: RecalculationPayload): ResolveByUuid<MetricErrorState> => {
const metricErrors = metricErrorEntries(recalculation)
const uuidsWithRow = new Set((recalculation.results ?? []).map((r) => r.metric_uuid))
const failedResultMessageByUuid = new Map(
(recalculation.results ?? [])
.filter((r) => r.status === 'failed' && r.error_message)
.map((r) => [r.metric_uuid, r.error_message as string])
)
return (uuid) => {
const entry =
metricErrors[uuid]?.error_type === RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE && uuidsWithRow.has(uuid)
? undefined
: metricErrors[uuid]
const message = entry?.message ?? failedResultMessageByUuid.get(uuid)
return message ? { detail: message, retriable: entry?.retriable === true } : null
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface experimentMetricsLogicValues {
isDev: boolean | undefined // preflightLogic
currentProjectId: number | null // projectLogic
backendEnforcesRefreshWindow: boolean
currentRecalculation: RecalculationPayload | null
isManualRefreshBlocked: boolean
isMetricRecalculating: (metricUuid: string | undefined) => boolean
isRecalculating: boolean
lastRefresh: string | null
liveRowsProgress: {
estimatedRows: number
recalculationId: string
rowsRead: number
} | null
metricRetries: Record<string, MetricRetryInfo>
nextAllowedManualRefresh: string | null
nextRetryAt: string | null
primaryMetricsResults: CachedNewExperimentQueryResponse[]
primaryMetricsResultsErrors: (unknown | null)[]
queuedRerun: ExperimentMetricsRecalculationRequestTriggerEnumApi | null
recalculatingMetricUuids: string[]
recalculationAbandoned: boolean
recalculationDisplayState: 'cold' | 'initial' | 'partial' | 'refreshing' | 'resting'
recalculationLoading: boolean
recalculationProgress: {
completed: number
total: number
}
recalculationRejectionMessage: string | null
refreshEligibilityTick: number
secondaryMetricsResults: CachedNewExperimentQueryResponse[]
secondaryMetricsResultsErrors: (unknown | null)[]
totalMetricsCount: number
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface experimentMetricsLogicActions {
abandonRecalculation: (
reason: RecalculationAbandonReason,
recalculationId: string | null
) => {
reason: RecalculationAbandonReason
recalculationId: string | null
}
loadLatestRecalculation: () => {
value: true
}
markRefreshWindowEnforced: () => {
value: true
}
pollRecalculation: (recalculationId: string) => {
recalculationId: string
}
recheckRefreshEligibility: () => {
value: true
}
setCurrentRecalculation: (recalculation: RecalculationPayload | null) => {
recalculation: RecalculationPayload | null
}
setPrimaryMetricsResults: (results: CachedNewExperimentQueryResponse[]) => {
results: CachedNewExperimentQueryResponse[]
}
setPrimaryMetricsResultsErrors: (errors: (unknown | null)[]) => {
errors: unknown[]
}
setQueuedRerun: (trigger: ExperimentMetricsRecalculationRequestTriggerEnumApi | null) => {
trigger: ExperimentMetricsRecalculationRequestTriggerEnumApi | null
}
setRecalculatingMetricUuids: (uuids: string[]) => {
uuids: string[]
}
setRecalculationLoading: (loading: boolean) => {
loading: boolean
}
setSecondaryMetricsResults: (results: CachedNewExperimentQueryResponse[]) => {
results: CachedNewExperimentQueryResponse[]
}
setSecondaryMetricsResultsErrors: (errors: (unknown | null)[]) => {
errors: unknown[]
}
triggerRecalculation: (trigger?: ExperimentMetricsRecalculationRequestTriggerEnumApi) => {
trigger: ExperimentMetricsRecalculationRequestTriggerEnumApi
}
}
// Generated by kea-typegen. Update if you're an agent, ignore if you're human.
export interface experimentMetricsLogicMeta {
key: ExperimentIdType
__keaTypeGenInternalSelectorTypes: {
isRecalculating: (
recalculationLoading: boolean,
currentRecalculation: RecalculationPayload | null,
recalculationAbandoned: boolean
) => boolean
recalculationProgress: (currentRecalculation: RecalculationPayload | null) => {
completed: number
total: number
}
totalMetricsCount: (arg: any) => number
recalculationRejectionMessage: (currentRecalculation: RecalculationPayload | null) => string | null
lastRefresh: (currentRecalculation: RecalculationPayload | null) => string | null
nextAllowedManualRefresh: (currentRecalculation: RecalculationPayload | null) => string | null
isManualRefreshBlocked: (
nextAllowedManualRefresh: string | null,
refreshEligibilityTick: number,
isDev: boolean | undefined
) => boolean
metricRetries: (currentRecalculation: RecalculationPayload | null) => Record<string, MetricRetryInfo>
nextRetryAt: (metricRetries: Record<string, MetricRetryInfo>) => string | null
recalculationDisplayState: (
recalculationLoading: boolean,
currentRecalculation: RecalculationPayload | null,
recalculationAbandoned: boolean
) => 'cold' | 'initial' | 'partial' | 'refreshing' | 'resting'
isMetricRecalculating: (recalculatingMetricUuids: string[]) => (metricUuid: string | undefined) => boolean
}
}
export type experimentMetricsLogicType = MakeLogicType<
experimentMetricsLogicValues,
experimentMetricsLogicActions,
ExperimentMetricsLogicProps,
experimentMetricsLogicMeta
>
export const experimentMetricsLogic = kea<experimentMetricsLogicType>([
props({} as ExperimentMetricsLogicProps),
key((props) => props.experiment.id),
path((key) => ['scenes', 'experiment', 'experimentMetricsLogic', String(key)]),
connect(() => ({
values: [projectLogic, ['currentProjectId'], preflightLogic, ['isDev']],
})),
actions({
setCurrentRecalculation: (recalculation: RecalculationPayload | null) => ({ recalculation }),
loadLatestRecalculation: true,
triggerRecalculation: (trigger: ExperimentMetricsRecalculationRequestTriggerEnumApi = 'manual') => ({
trigger,
}),
pollRecalculation: (recalculationId: string) => ({ recalculationId }),
recheckRefreshEligibility: true,
markRefreshWindowEnforced: true,
setPrimaryMetricsResults: (results: CachedNewExperimentQueryResponse[]) => ({ results }),
setSecondaryMetricsResults: (results: CachedNewExperimentQueryResponse[]) => ({ results }),
setPrimaryMetricsResultsErrors: (errors: (unknown | null)[]) => ({ errors }),
setSecondaryMetricsResultsErrors: (errors: (unknown | null)[]) => ({ errors }),
setRecalculationLoading: (loading: boolean) => ({ loading }),
// The metrics still showing a stale value while a non-cold recalc refreshes them in place.
setRecalculatingMetricUuids: (uuids: string[]) => ({ uuids }),
setQueuedRerun: (trigger: ExperimentMetricsRecalculationRequestTriggerEnumApi | null) => ({ trigger }),
abandonRecalculation: (reason: RecalculationAbandonReason, recalculationId: string | null) => ({
reason,
recalculationId,
}),
}),
reducers({
currentRecalculation: [
null as RecalculationPayload | null,
{
setCurrentRecalculation: (_, { recalculation }) => recalculation,
},
],
recalculationLoading: [
false,
{
setRecalculationLoading: (_, { loading }) => loading,
loadLatestRecalculation: () => true,
setCurrentRecalculation: () => false,
},
],
// The page stopped waiting on the current run, so its pending or in_progress status no longer means loading.
recalculationAbandoned: [
false,
{
abandonRecalculation: () => true,
loadLatestRecalculation: () => false,
setCurrentRecalculation: () => false,
},
],
// Bumped when the manual refresh window closes, so isManualRefreshBlocked recomputes without new data.
refreshEligibilityTick: [0, { recheckRefreshEligibility: (state: number) => state + 1 }],
// A 429 proves the backend enforces the window, even when this page's cached flags still say otherwise.
backendEnforcesRefreshWindow: [false, { markRefreshWindowEnforced: () => true }],
queuedRerun: [
null as ExperimentMetricsRecalculationRequestTriggerEnumApi | null,
{
/**
* If the state is `experiment_config_change`, it sticks.
*/
setQueuedRerun: (state, { trigger }) =>
trigger === null ? null : state === 'experiment_config_change' ? state : trigger,
},
],
primaryMetricsResults: [
[] as CachedNewExperimentQueryResponse[],
{
setPrimaryMetricsResults: (_, { results }) => results,
},
],
secondaryMetricsResults: [
[] as CachedNewExperimentQueryResponse[],
{
setSecondaryMetricsResults: (_, { results }) => results,
},
],
primaryMetricsResultsErrors: [
[] as (unknown | null)[],
{
setPrimaryMetricsResultsErrors: (_, { errors }) => errors,
},
],
secondaryMetricsResultsErrors: [
[] as (unknown | null)[],
{
setSecondaryMetricsResultsErrors: (_, { errors }) => errors,
},
],
recalculatingMetricUuids: [
[] as string[],
{
setRecalculatingMetricUuids: (_, { uuids }) => uuids,
},
],
liveRowsProgress: [
null as { recalculationId: string; rowsRead: number; estimatedRows: number } | null,
{
setCurrentRecalculation: (state, { recalculation }) => {
if (!recalculation) {
return null
}
const rowsRead = recalculation.rows_read ?? 0
if (rowsRead > 0) {
return {
recalculationId: recalculation.id,
rowsRead,
estimatedRows: recalculation.estimated_rows_total ?? 0,
}
}
return state?.recalculationId === recalculation.id ? state : null
},
},
],
}),
selectors({
// True while a recalculation is being fetched or is still running.
isRecalculating: [
(s) => [s.recalculationLoading, s.currentRecalculation, s.recalculationAbandoned],
(
recalculationLoading: boolean,
recalculation: RecalculationPayload | null,
recalculationAbandoned: boolean
): boolean =>
recalculationLoading ||
(!recalculationAbandoned &&
(recalculation?.status === RECALCULATION_STATUSES.pending ||
recalculation?.status === RECALCULATION_STATUSES.in_progress)),
],
recalculationProgress: [
(s) => [s.currentRecalculation],
// "completed" here means resolved: a failed metric is done too, so it counts toward progress.
// Without this, a run where every metric fails sits at 0/N forever and looks stuck.
(recalc: RecalculationPayload | null): { completed: number; total: number } => ({
completed: (recalc?.completed_metrics ?? 0) + (recalc?.failed_metrics ?? 0),
total: recalc?.total_metrics ?? 0,
}),
],
// Total metrics on the experiment itself, independent of whether a recalculation run exists yet.
// Sourced from the experiment so the count is truthful on first render, before the first run loads.
totalMetricsCount: [
() => [(_, props) => props.experiment],
(experiment: Experiment): number =>
metricsInOrder(experiment, 'primary').length + metricsInOrder(experiment, 'secondary').length,
],
recalculationRejectionMessage: [
(s) => [s.currentRecalculation],
(recalc: RecalculationPayload | null): string | null => (recalc ? rejectionMessage(recalc) : null),
],
lastRefresh: [
(s) => [s.currentRecalculation],
(recalc: RecalculationPayload | null): string | null => recalc?.query_to ?? null,
],
nextAllowedManualRefresh: [
(s) => [s.currentRecalculation],
// Only a completed run anchors the window: the timeseries fallback is not a run, and a failed
// run must stay reloadable. The window measures from completed_at, as the backend does.
(recalc: RecalculationPayload | null): string | null =>
recalc?.status === RECALCULATION_STATUSES.completed &&
recalc.result_source !== 'timeseries_fallback' &&
recalc.completed_at
? dayjs(recalc.completed_at).add(MIN_MANUAL_REFRESH_INTERVAL_MINUTES, 'minutes').toISOString()
: null,
],
isManualRefreshBlocked: [
(s) => [s.nextAllowedManualRefresh, s.refreshEligibilityTick, s.isDev],
// State for the reload button only. The backend is the gate: it answers 429 inside the window.
// Local development skips the window, as the backend does, so a developer can reload at will.
(nextAllowedManualRefresh: string | null, _tick: number, isDev: boolean | undefined): boolean =>
!isDev && !!nextAllowedManualRefresh && dayjs(nextAllowedManualRefresh).isAfter(dayjs()),
],
metricRetries: [
(s) => [s.currentRecalculation],
(recalc: RecalculationPayload | null): Record<string, MetricRetryInfo> => {
const raw = (recalc?.metric_retries ?? {}) as Record<string, MetricRetryInfo>
const landed = new Set([
...(recalc?.results ?? []).map(({ metric_uuid }) => metric_uuid),
...Object.keys((recalc?.metric_errors as Record<string, unknown> | null) ?? {}),
])
return Object.fromEntries(Object.entries(raw).filter(([uuid]) => !landed.has(uuid)))
},
],
nextRetryAt: [
(s) => [s.metricRetries],
(metricRetries: Record<string, MetricRetryInfo>): string | null => {
const upcoming = Object.values(metricRetries)
.map(({ next_retry_at }) => next_retry_at)
.filter((value): value is string => !!value)
.sort()
return upcoming[0] ?? null
},
],
recalculationDisplayState: [
(s) => [s.recalculationLoading, s.currentRecalculation, s.recalculationAbandoned],
(
recalculationLoading: boolean,
recalculation: RecalculationPayload | null,
recalculationAbandoned: boolean
): 'initial' | 'cold' | 'refreshing' | 'partial' | 'resting' => {
if (!recalculation) {
return recalculationLoading ? 'initial' : 'resting'
}
const inFlight =
recalculationLoading ||
(!recalculationAbandoned &&
(recalculation.status === RECALCULATION_STATUSES.pending ||
recalculation.status === RECALCULATION_STATUSES.in_progress))
if (inFlight) {
return (recalculation.results?.length ?? 0) > 0 ? 'refreshing' : 'cold'
}
return recalculation.failed_metrics > 0 ? 'partial' : 'resting'
},
],
// Predicate the table uses to dim a metric whose stale value is still being refreshed.
isMetricRecalculating: [
(s) => [s.recalculatingMetricUuids],
(uuids: string[]): ((metricUuid: string | undefined) => boolean) => {
const recalculating = new Set(uuids)
return (metricUuid) => !!metricUuid && recalculating.has(metricUuid)
},
],
}),
listeners(({ actions, values, props, cache }) => {
const experimentIsLaunched = (): boolean => isLaunched(props.experiment)
const markResultsReviewed = (recalculation: RecalculationPayload): void => {
const terminal =
recalculation.status === RECALCULATION_STATUSES.completed ||
recalculation.status === RECALCULATION_STATUSES.failed
if (terminal && experimentMetricsLogic.findMounted(props) && experimentIsLaunched()) {
globalSetupLogic.findMounted()?.actions.markTaskAsCompleted(SetupTaskId.ReviewExperimentResults)
}
}
const ids = (): { projectId: number; experimentId: number } | null => {
const projectId = values.currentProjectId
const experimentId = props.experiment.id
if (!projectId || typeof experimentId !== 'number') {
return null
}
return { projectId, experimentId }
}
const emitTerminalEvent = (recalculation: RecalculationPayload): void => {
const startMs = cache.recalcStartMs ?? Date.now()
reportExperimentMetricRecalculation(
recalculation.status === RECALCULATION_STATUSES.completed ? 'completed' : 'failed',
{
experiment_id: props.experiment.id as number,
recalculation_id: recalculation.id,
total_metrics: recalculation.total_metrics,
succeeded: recalculation.completed_metrics,
failed: recalculation.failed_metrics,
duration_ms: Date.now() - startMs,
poll_count: cache.pollCount ?? 0,
}
)
}
/**
* Gives every metric with neither a result nor an error a retryable error. Without one, the row reads as
* loading (no result, no error) for as long as the page stays open.
*/
const markUnresolvedMetricsRetriable = (detail: string): void => {
const error: MetricErrorState = { detail, retriable: true }
const withRetriableErrors = (
type: 'primary' | 'secondary',
results: readonly (CachedNewExperimentQueryResponse | undefined)[],
errors: readonly (unknown | null)[]
): (unknown | null)[] =>
metricsInOrder(props.experiment, type).map(
(_, index) => errors[index] ?? (results[index] ? null : error)
)
actions.setPrimaryMetricsResultsErrors(
withRetriableErrors('primary', values.primaryMetricsResults, values.primaryMetricsResultsErrors)
)
actions.setSecondaryMetricsResultsErrors(
withRetriableErrors('secondary', values.secondaryMetricsResults, values.secondaryMetricsResultsErrors)
)
}
const applyResults = (recalculation: RecalculationPayload): void => {
const resultFor = resolveResultByUuid(recalculation.results)
const errorFor = resolveErrorByUuid(recalculation)
const alignPrimary = alignByMetricPosition(props.experiment, 'primary')
const alignSecondary = alignByMetricPosition(props.experiment, 'secondary')
const nextResults = alignPrimary(resultFor)
const nextSecondaryResults = alignSecondary(resultFor)
const nextErrors = alignPrimary(errorFor)
const nextSecondaryErrors = alignSecondary(errorFor)
const mergeResults = (
current: CachedNewExperimentQueryResponse[],
nextResult: (CachedNewExperimentQueryResponse | undefined)[],
nextError: MetricErrorState[]
): CachedNewExperimentQueryResponse[] =>
nextResult.map((value, i) => {
if (value !== undefined) {
return value
}
// A fresh error for this slot supersedes any previously shown result.
if (nextError[i]) {
return undefined as unknown as CachedNewExperimentQueryResponse
}
return current[i]
})
const mergeErrors = (
current: (unknown | null)[],
nextError: MetricErrorState[],
nextResult: (CachedNewExperimentQueryResponse | undefined)[]
): (unknown | null)[] =>
nextError.map((value, i) => {
if (value) {
return value
}
// A fresh result for this slot clears any previously shown error.
if (nextResult[i] !== undefined) {
return null
}
return current[i] ?? null
})
actions.setPrimaryMetricsResults(mergeResults(values.primaryMetricsResults, nextResults, nextErrors))
actions.setSecondaryMetricsResults(
mergeResults(values.secondaryMetricsResults, nextSecondaryResults, nextSecondaryErrors)
)
actions.setPrimaryMetricsResultsErrors(
mergeErrors(values.primaryMetricsResultsErrors, nextErrors, nextResults)
)
actions.setSecondaryMetricsResultsErrors(
mergeErrors(values.secondaryMetricsResultsErrors, nextSecondaryErrors, nextSecondaryResults)
)
if (values.recalculatingMetricUuids.length > 0) {
const landed = new Set([
// metrics that produced a fresh result this poll
...(recalculation.results ?? []).map(({ metric_uuid }) => metric_uuid),
// metrics that failed this poll
...Object.keys((recalculation.metric_errors as Record<string, unknown> | null) ?? {}),
])
// Clear the tag on each metric whose fresh result or failure just landed; the rest keep it until they do.
actions.setRecalculatingMetricUuids(values.recalculatingMetricUuids.filter((uuid) => !landed.has(uuid)))
}
}
return {
loadLatestRecalculation: async () => {
// Don't fetch for draft experiments — there's nothing to recalculate yet.
if (!experimentIsLaunched()) {
actions.setRecalculationLoading(false)
return
}
/**
* guard against invalid project or experiment. bail and clear recalculation
* loading state
*/
const resolvedIds = ids()
if (!resolvedIds) {
actions.setRecalculationLoading(false)
return
}
try {
const { projectId, experimentId } = resolvedIds
/**
* retrieve the latest complete recalculation so we always show results, even if stale.
* store it in state and apply the results.
*/
const recalculation = await withRequestTimeout(
(signal) =>
experimentsMetricsRecalculationLatestRetrieve(String(projectId), experimentId, { signal }),
LATEST_REQUEST_TIMEOUT_MS
)
if (cache.disposables.isDisposed) {
return
}
actions.setCurrentRecalculation(recalculation)
applyResults(recalculation)
markResultsReviewed(recalculation)
/**
* if there's an active recalculation running, schedule polling of
* the active recalculation status
*/
if (recalculation.active_run) {
actions.setRecalculatingMetricUuids(
metricUuidsToMarkRecalculating(
props.experiment,
values.primaryMetricsResults,
values.secondaryMetricsResults,
values.primaryMetricsResultsErrors,
values.secondaryMetricsResultsErrors
)
)
cache.activeRecalculationId = recalculation.active_run.id
cache.recalcStartMs = Date.now()
cache.pollCount = 0
cache.pollRetryCount = 0
actions.pollRecalculation(recalculation.active_run.id)
return
}
/**
* Fill any gap in a terminal latest (a real run, a failed run, or the timeseries fallback);
* otherwise the uncovered metric shows a perpetual loading state, since nothing else re-runs
* on page load. What is already shown stays visible and cells update in place as the run polls.
*/
/** A rejected run gets the same answer on every heal, so a heal would only loop. */
if (rejectionMessage(recalculation)) {
return
}
/**
* A stale refresh advances the window over every metric, so it also fills a gap. It runs once
* per mount, so a run that cannot advance the window does not start a new run on every load.
*/
if (!cache.staleRefreshRequested && resultsAreStale(props.experiment, recalculation)) {
cache.staleRefreshRequested = true
actions.triggerRecalculation('stale_refresh')
} else if (recalculationHasGap(props.experiment, recalculation)) {
actions.triggerRecalculation(HEAL_TRIGGER_BY_RESULT_SOURCE[recalculation.result_source])
}
} catch (error: any) {
if (cache.disposables.isDisposed) {
return
}
if (error?.status === 404) {
/**
* if there are no completed recalculations, kick off a new one.
* this should only run on page loads, so no need to load exposures.
*/
actions.triggerRecalculation('cold_run')
return
}
if (error instanceof RecalculationRequestTimeoutError) {
actions.abandonRecalculation('latest_timeout', null)
return
}
lemonToast.error("Couldn't load experiment results. Refresh the results to try again.")
markUnresolvedMetricsRetriable("Couldn't load results for this metric.")
} finally {
if (!cache.disposables.isDisposed) {
actions.setRecalculationLoading(false)
}
}
},
abandonRecalculation: ({ reason, recalculationId }) => {
actions.setRecalculatingMetricUuids([])
// A queued rerun drains only when a poll reaches terminal, which no longer happens.
actions.setQueuedRerun(null)
actions.setRecalculationLoading(false)
markUnresolvedMetricsRetriable(ABANDON_MESSAGES[reason])
lemonToast.error(ABANDON_MESSAGES[reason])
reportExperimentMetricRecalculation('abandoned', {
experiment_id: props.experiment.id as number,
recalculation_id: recalculationId,
reason,
...(recalculationId
? {
duration_ms: Date.now() - (cache.recalcStartMs ?? Date.now()),
poll_count: cache.pollCount ?? 0,
}
: {}),
})
},
setCurrentRecalculation: () => {
// Re-enable the reload button the moment the window closes, not on the next data load.
cache.disposables.dispose('manualRefreshEligibility')
const nextAllowed = values.nextAllowedManualRefresh
if (!nextAllowed || !dayjs(nextAllowed).isAfter(dayjs())) {
return
}
cache.disposables.add(() => {
const timerId = setTimeout(
actions.recheckRefreshEligibility,
Math.max(0, dayjs(nextAllowed).diff(dayjs())) + 100
)
return () => clearTimeout(timerId)
}, 'manualRefreshEligibility')
},
triggerRecalculation: async ({ trigger }) => {
/**
* Don't recalculate draft experiments; a config-change edit on a draft shouldn't kick off a
* run. Stopped experiments are allowed — they still need their final results computed.
*/
if (!experimentIsLaunched()) {
return
}
/**
* A config change while a run is active queues a rerun; the terminal branch (poll, or a
* terminal-on-create) drains it. Guard on the run status, not isRecalculating: a transient
* loadLatestRecalculation fetch also flips isRecalculating, and queueing against it strands the
* rerun forever, because that fetch never polls and so never drains the queue.
*
* `cache.createInFlight` closes the window between the create POST firing and
* setCurrentRecalculation landing the pending run: during that round-trip currentRecalculation
* still holds the previous terminal status, so without this a second config change would post
* again, and the backend would dedupe it to the active run and drop its trigger.
*/
const runStatus = values.currentRecalculation?.status
const runInFlight =
!values.recalculationAbandoned &&
(runStatus === RECALCULATION_STATUSES.pending || runStatus === RECALCULATION_STATUSES.in_progress)
if (
(trigger === 'experiment_config_change' || trigger === 'metric_config_change') &&
(runInFlight || cache.createInFlight)
) {
actions.setQueuedRerun(trigger)
return
}
const previousRecalculatingMetricUuids = new Set(values.recalculatingMetricUuids)
if (trigger !== 'cold_run') {
actions.setRecalculatingMetricUuids(
metricUuidsToMarkRecalculating(
props.experiment,
values.primaryMetricsResults,
values.secondaryMetricsResults,
values.primaryMetricsResultsErrors,
values.secondaryMetricsResultsErrors
)
)
}
/**
* guard against invalid project or experiment. bail and clear recalculation
* loading state
*/
const resolvedIds = ids()
if (!resolvedIds) {
return
}
/**
* Mark loading up front so the reload button disables on click, not only once the create POST
* returns. Without this there's a window (the POST round-trip) where the button stays clickable.
* Cleared by setCurrentRecalculation on success, or in the catch below on failure.
*/
actions.setRecalculationLoading(true)
// Mark the create in flight synchronously so a second config change during the POST round-trip
// queues instead of posting a duplicate the backend would dedupe and drop. Cleared in finally.
cache.createInFlight = true
try {
const { projectId, experimentId } = resolvedIds
/**
* 201 with a new pending run, or 200 with the already-active one. No results yet.
* Create a recalculation workflow. 201: a new run. 200: one is already running, poll it.
*/
const recalculation = await experimentsMetricsRecalculationCreate(String(projectId), experimentId, {
trigger,
})
/**
* Mark the active run. A terminal-on-create run (below) applies results without
* dispatching pollRecalculation, so breakpoint can't abort an older poll; pollRecalculation
* re-checks this id after its fetch and bails if a newer run took over.
*/
cache.activeRecalculationId = recalculation.id
/**
* Lifecycle clock + poll count for the terminal-state analytics event. Reset on every
* trigger so duration_ms / poll_count are anchored to THIS run.
*/
cache.recalcStartMs = Date.now()
cache.pollCount = 0
cache.pollRetryCount = 0
actions.setCurrentRecalculation(recalculation)
reportExperimentMetricRecalculation('triggered', {
experiment_id: experimentId,
recalculation_id: recalculation.id,
trigger,
is_existing: recalculation.is_existing,
})
if (
recalculation.status === RECALCULATION_STATUSES.completed ||
recalculation.status === RECALCULATION_STATUSES.failed
) {
/**
* Create can return an already-terminal run (e.g. a completed one finished between
* request and response). Load its results directly rather than polling. A rejected
* run has no results on the create response, so latest serves the earlier ones. The
* same applies to an existing run the backend returns instead of a stale refresh.
*/
if (rejectionMessage(recalculation) || recalculation.is_existing) {
/** A failed reload never reaches applyResults, so the marks of this create clear here. */
actions.setRecalculatingMetricUuids(
values.recalculatingMetricUuids.filter(
(uuid) => !(uuid in metricErrorEntries(recalculation))
)
)
actions.loadLatestRecalculation()
} else {
applyResults(recalculation)
}
emitTerminalEvent(recalculation)
// Terminal-on-create arms no poll, so drain any rerun queued during this create here,
// the way the poll's terminal branch does. Otherwise a config change made mid-create
// would strand. Clear createInFlight first so the drained rerun creates instead of
// re-queuing against this finished create.
cache.createInFlight = false
const queued = values.queuedRerun
if (queued) {
actions.setQueuedRerun(null)
actions.triggerRecalculation(queued)
}
} else {
if (trigger === 'manual' && !recalculation.is_existing) {
lemonToast.info(
'Recalculating metrics in the background. Results will update as they finish.'
)
}
actions.pollRecalculation(recalculation.id)
}
} catch (error: any) {
/**
* re enable the reload button.
*/
actions.setRecalculationLoading(false)
/**
* clears the metric badges and the metric group loading indicator this create set,
* and keeps the marks of a run that is already being polled. The intersection, not the
* snapshot: a poll that landed while the request was pending has already cleared its
* metrics, and restoring them would leave them on with nothing left to clear them.
*/
actions.setRecalculatingMetricUuids(
values.recalculatingMetricUuids.filter((uuid) => previousRecalculatingMetricUuids.has(uuid))
)
if (error?.status === 429) {
// Another tab or an agent used the refresh window first. Reload the latest run so the
// button picks up its completed_at and shows when the next refresh is possible.
lemonToast.info(error?.detail || 'Metrics were recalculated less than 5 minutes ago.')
actions.markRefreshWindowEnforced()
actions.loadLatestRecalculation()
return
}
lemonToast.error(error?.detail || 'Failed to trigger metrics recalculation')
} finally {
cache.createInFlight = false
}
},
/**
* Polls one run to terminal status, re-arming one tick at a time rather than looping in place.
* `breakpoint` is kea's cancellation primitive: `await breakpoint(ms)` paces the poll and throws
* if pollRecalculation re-fires or the logic unmounts; `breakpoint()` re-checks without delaying.
* So a newer poll auto-cancels this one's in-flight breakpoint before it can write stale results:
* free poll-vs-poll supersession, no interval to clear. The gap: a terminal-on-create run applies
* results without dispatching pollRecalculation, so cache.activeRecalculationId covers that case.
*/
pollRecalculation: async ({ recalculationId }, breakpoint) => {
const resolvedIds = ids()
if (!resolvedIds) {
return
}
const { projectId, experimentId } = resolvedIds
// First tick of a new run (the active id changed): reset the retry streak so a prior run that
// exhausted its retries doesn't leave the counter at the cap for this one.
if (cache.activeRecalculationId !== recalculationId) {
cache.pollRetryCount = 0
}
cache.activeRecalculationId = recalculationId
/**
* if for any reason the recalculation workflow never reaches terminal state, we need to
* stop polling, and clear recalculation state. This includes any queued rerun, that will never
* be fired.
*/
const runAgeMs = Date.now() - (cache.recalcStartMs ?? Date.now())
if (runAgeMs > MAX_POLL_DURATION_MS) {
actions.abandonRecalculation('poll_duration_cap', recalculationId)
return
}
// Pace this tick; aborts here if a newer poll superseded us or the logic unmounted.
await breakpoint(pollIntervalMs(runAgeMs))
let recalculation: ExperimentMetricsRecalculationRunApi
try {
recalculation = await withRequestTimeout(
(signal) =>
experimentsMetricsRecalculationRetrieve(String(projectId), experimentId, recalculationId, {
signal,
}),
POLL_REQUEST_TIMEOUT_MS
)
} catch (error) {
// A slow request can outlive a newer poll; don't count or abandon on behalf of a superseded run.
breakpoint()
if (cache.activeRecalculationId !== recalculationId) {
return
}
/**
* Retry on transient errors and timeouts, but give up after MAX_POLL_RETRIES consecutive
* failures so a persistently failing endpoint can't poll forever.
*/
cache.pollRetryCount = (cache.pollRetryCount ?? 0) + 1
if (cache.pollRetryCount >= MAX_POLL_RETRIES) {
actions.abandonRecalculation(
error instanceof RecalculationRequestTimeoutError ? 'poll_timeout' : 'poll_failed',
recalculationId
)
return
}
actions.pollRecalculation(recalculationId)
return
}
// Healthy response; reset the retry streak so an earlier blip doesn't count against later ticks.
cache.pollRetryCount = 0
// A newer poll may have started while the request was in flight; abort before writing results.
breakpoint()
/**
* breakpoint covers a newer POLL; this covers a newer terminal-on-create run that applied
* results directly without dispatching pollRecalculation (so breakpoint never fired for us).
*/
if (cache.activeRecalculationId !== recalculationId) {
return
}
/**
* Anchor duration telemetry (and the zombie cap) to the run's actual start, not the moment
* this tab triggered or resumed it. Once per run: the first successful poll wins.
*/
if (cache.recalcAnchoredId !== recalculationId) {
const runStartMs = Date.parse(recalculation.started_at ?? recalculation.created_at)
if (!Number.isNaN(runStartMs)) {
cache.recalcStartMs = runStartMs
}
cache.recalcAnchoredId = recalculationId
}
/**
* Count only polls that pass the supersession check so the analytics event reflects the work
* THIS run actually paid for; superseded ticks would otherwise inflate the number.
*/
cache.pollCount = (cache.pollCount ?? 0) + 1
actions.setCurrentRecalculation(recalculation)
if (
recalculation.status === RECALCULATION_STATUSES.pending ||
recalculation.status === RECALCULATION_STATUSES.in_progress
) {
/**
* Surface each metric as it lands rather than waiting for terminal. applyResults is
* positional and idempotent, and it un-dims each recalculating metric as its result
* arrives, so the table streams fresh values in place.
*/
applyResults(recalculation)
actions.pollRecalculation(recalculationId)
return
}
// Successful metrics load, failed metrics surface their error in-row.
applyResults(recalculation)
markResultsReviewed(recalculation)
emitTerminalEvent(recalculation)
if (recalculation.failed_metrics > 0) {
lemonToast.error(
`${recalculation.failed_metrics} of ${recalculation.total_metrics} metrics failed to load`
)
}
/**
* The user has changed the experiment or the metrics configuration while a recalculation was in flight.
* We clear the flag an trigger a new recalculation to reflect the changes. Cache resolution is handled by the
* durable execution backend, so we'll try to reuse the cached results.
*/
const queued = values.queuedRerun
if (queued) {
actions.setQueuedRerun(null)
actions.triggerRecalculation(queued)
}
},
}
}),
afterMount(({ actions, values, cache }) => {
// Fetch the latest completed recalculation on mount.
actions.loadLatestRecalculation()
/**
* On tab visible, re-fetch the latest only when nothing is loaded: refetching would overwrite a
* displayed failure state with the last completed run.
*/
cache.disposables.add(
() => {
const handler = (): void => {
if (document.visibilityState === 'visible' && !values.currentRecalculation) {
actions.loadLatestRecalculation()
}
}
document.addEventListener('visibilitychange', handler)
return () => document.removeEventListener('visibilitychange', handler)
},
'recalculationVisibility',
{ pauseOnPageHidden: false }
)
}),
])