:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from typing import Final, Literal
from posthog.schema import PrecomputationMode
from posthog.dataclasses import frozen
# Shared by the workflow definition, the schedule, and the management command.
CANARY_WORKFLOW_NAME = "experiment-precompute-canary"
CanaryOutcome = Literal["pass", "divergence", "path_flip", "uncheckable", "error", "skipped"]
OUTCOME_PASS: Final = "pass"
OUTCOME_DIVERGENCE: Final = "divergence"
OUTCOME_PATH_FLIP: Final = "path_flip"
# The direct-scan ground truth cannot execute under the per-query byte cap, so correctness is
# unverifiable for this metric. Stability was still checked. Kept separate from "error" so the
# error gauge only counts unexpected failures.
OUTCOME_UNCHECKABLE: Final = "uncheckable"
OUTCOME_ERROR: Final = "error"
OUTCOME_SKIPPED: Final = "skipped"
ALL_OUTCOMES: tuple[CanaryOutcome, ...] = (
OUTCOME_PASS,
OUTCOME_DIVERGENCE,
OUTCOME_PATH_FLIP,
OUTCOME_UNCHECKABLE,
OUTCOME_ERROR,
OUTCOME_SKIPPED,
)
# Cap CanaryMetricResult.detail so a pathological error message can't bloat the Temporal payload.
MAX_CANARY_DETAIL_LENGTH = 1000
# Open executions and the workflow_type metric label carry this name, so a rename strands running workflows.
METRICS_RECALCULATION_WORKFLOW_NAME = "experiment-metrics-recalculation-workflow"
# Max attempts per metric before it's marked failed on the recalculation workflow.
MAX_METRIC_ATTEMPTS = 8
# Below MAX_METRIC_ATTEMPTS for errors where a retry rarely succeeds and adds ClickHouse load. At the cap the
# activity fails the metric non-retryable, so the RetryPolicy keeps MAX_METRIC_ATTEMPTS.
# A capacity rejection costs nothing, because ClickHouse refuses the query before it runs, so it gets more
# attempts than a timeout. 6 attempts are 5 waits, one per band of the CAPACITY_RETRY_* schedule below.
CAPACITY_MAX_METRIC_ATTEMPTS = 6
# The timeout class also holds SOCKET_TIMEOUT, so one retry stays for network failures.
TIMEOUT_MAX_METRIC_ATTEMPTS = 2
# Delay before the retry of an attempt that a capacity rejection ended (ClickHouseAtCapacity or
# ConcurrencyLimitExceeded), set via next_retry_delay. The delay is a random value in a band, and the band
# doubles with each attempt: 10-20 s, 20-40 s, 40-80 s, 80-160 s, then 160-300 s. The short first wait fits the
# common case, a pool that frees up within seconds. The later, longer waits stop a sustained full pool from
# being polled constantly. The random position in the band spreads out metrics that bounced in the same burst,
# so their retries do not arrive together. The per-org app limiter skips Temporal activities, so this delay is
# what lets the cluster drain. The sum of the bands, at most 600 s, stays inside
# METRIC_CALC_SCHEDULE_TO_CLOSE_TIMEOUT_SECONDS where that budget applies.
CAPACITY_RETRY_FIRST_DELAY_MIN_SECONDS = 10
CAPACITY_RETRY_FIRST_DELAY_MAX_SECONDS = 20
CAPACITY_RETRY_BACKOFF_COEFFICIENT = 2.0
CAPACITY_RETRY_MAX_DELAY_SECONDS = 300
RECALCULATION_RETRY_INITIAL_INTERVAL_SECONDS = 5
RECALCULATION_RETRY_BACKOFF_COEFFICIENT = 2.0
RECALCULATION_RETRY_MAX_INTERVAL_SECONDS = 60
# classify_experiment_query_error classes that are deterministic for a fixed query and data window: retrying
# burns a full ClickHouse query per attempt without changing the outcome, so the activity fails non-retryable
# and persists immediately. timeout is deliberately absent — the class conflates TIMEOUT_EXCEEDED with
# SOCKET_TIMEOUT (a transient network blip), and recalc timeouts are rare enough to keep the retry budget.
# The window type comes from request_recalculation, not from a query: the experiment start date sets it.
RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE = "recalculation_window_too_long"
NON_RETRYABLE_ERROR_TYPES = frozenset(
{"out_of_memory", "byte_limit", "validation_error", RECALCULATION_WINDOW_TOO_LONG_ERROR_TYPE}
)
# Per-attempt ceiling for the calc activity (its start_to_close_timeout). When it fires, Temporal ends the
# attempt from the outside. The next heartbeat delivers the cancellation, which kills the ClickHouse query, but
# the attempt persists nothing. If no further attempt follows, the workflow failure backstop persists it.
METRIC_CALC_ACTIVITY_TIMEOUT_SECONDS = 300
# Total budget for one metric across all its attempts and retry delays, queue wait included (the calc
# activity's schedule_to_close_timeout). It keeps the IN_PROGRESS phase (this budget, the failure backstop and
# the finish step) below the staleness TTL in recalculation.py, so a slow but healthy run is never treated as
# stale. The budget ends a metric before MAX_METRIC_ATTEMPTS when its attempts are slow. Only a calc on the
# workflow's own queue gets it. A calc on the activity or precompute queue waits for a slot with no total budget,
# so its run can outlast the staleness TTL.
METRIC_CALC_SCHEDULE_TO_CLOSE_TIMEOUT_SECONDS = 1200
# Without a heartbeat, Temporal can deliver a cancellation to the calc activity only after start_to_close,
# and it detects a dead worker only then. The activity heartbeats every heartbeat_timeout / 4.
METRIC_CALC_HEARTBEAT_TIMEOUT_SECONDS = 60
# ClickHouse max_execution_time for the recalc metric query. Deliberately below
# METRIC_CALC_ACTIVITY_TIMEOUT_SECONDS (minus headroom for metric build, stats, and persistence) so a slow
# query fails INSIDE the activity with a typed ClickHouse error (159 TIMEOUT_EXCEEDED) instead of the activity
# being killed by Temporal — which would lose the error type, the FAILED result row, and the terminal event.
# A query needing more than this was already doomed to lose its attempt at the 5-minute kill; failing fast
# also stops the orphaned query from burning ClickHouse for the full default 600s.
METRIC_CALC_MAX_EXECUTION_TIME_SECONDS = 270
RECALCULATION_PROGRESS_ACTIVITY_TIMEOUT_SECONDS = 90
RECALCULATION_PROGRESS_MAX_ATTEMPTS = 3
# The failure backstop does only Postgres writes and one capture. Its worst case adds to the IN_PROGRESS phase,
# so it stays short. The schedule_to_close bound also covers queue wait, which start_to_close does not.
METRIC_FAILURE_BACKSTOP_ACTIVITY_TIMEOUT_SECONDS = 30
METRIC_FAILURE_BACKSTOP_MAX_ATTEMPTS = 3
METRIC_FAILURE_BACKSTOP_SCHEDULE_TO_CLOSE_TIMEOUT_SECONDS = 120
@dataclasses.dataclass(frozen=False)
class ExperimentMetricsRecalculationWorkflowInputs:
"""Input to the batch metrics recalculation workflow."""
recalculation_id: str # UUID as string
# Task queue for the metric calculation activity, chosen when the run starts so replay stays deterministic.
# None keeps the activity on the workflow's own queue.
activity_task_queue: str | None = None
# The key version that the run files its results under, chosen when the run is requested.
calculation_key_version: int = 1
@dataclasses.dataclass(frozen=False)
class ExperimentMetricToRecalculate:
"""A single metric to recalculate.
The recalc fingerprint is derived inside the calculate activity from the recalculation_id,
so it is intentionally not carried here.
"""
experiment_id: int
metric_uuid: str
# "primary" or "secondary": the metric's role in the experiment, not its kind.
role: str
# The calc activity's queue and precompute mode, chosen at discovery. The workflow uses them only for a run
# that has an activity_task_queue. None keeps the run's activity_task_queue and the AUTO mode.
task_queue: str | None = None
precomputation_mode: PrecomputationMode | None = None
@dataclasses.dataclass
class MetricRecalculationResult:
"""Result from calculating a single metric.
Lightweight by design: carries only success/error metadata, never the computed result blob. The blob is
written to ExperimentMetricResult (Postgres) inside the activity and read back by the API, so it never
crosses the Temporal payload boundary (~2 MiB cap). error_message is capped by the activity before being set.
"""
metric_uuid: str
success: bool
error_step: str | None = None
error_message: str | None = None
@frozen
class MetricOutOfTimeFailure:
"""A metric whose calc activity Temporal ended from the outside, so no attempt persisted its outcome."""
recalculation_id: str
experiment_id: int
metric_uuid: str
role: str # "primary" or "secondary"
calculation_key_version: int = 1
@dataclasses.dataclass
class ExperimentPrecomputeCanaryInputs:
"""Input to the precompute canary workflow.
Scheduled runs use the defaults (quota sampling across eligible experiments). On-demand forensics runs
set experiment_id to canary one specific experiment — all of its eligible metrics, quotas ignored —
optionally narrowed via metric_uuids. triggered_manually skips the Prometheus push (manual runs would
distort the scheduled-canary health signal) but still posts to Slack.
"""
experiment_id: int | None = None
metric_uuids: list[str] | None = None
funnel_quota: int = 12
# Bake-in level for the new mean metric-events precompute path: ~40% of sampled means are
# CUPED/breakdown/DW-ineligible and exercise nothing new, so 20 nominal ≈ 12 effective.
# Drop back toward ~10 once the mean table has a few clean weeks of canary history.
mean_quota: int = 20
ratio_quota: int = 4
# Bake-in level for the retention metric-events precompute path. Most retention metrics are
# eligible (breakdowns and data warehouse sources are rare on them), so nominal stays close
# to effective. Drop toward ~5 once retention has a few clean weeks of canary history.
retention_quota: int = 15
per_experiment_cap: int = 3
time_budget_seconds: int = 5400
triggered_manually: bool = False
@dataclasses.dataclass
class CanaryMetricTarget:
"""One sampled (experiment, metric) pair. Carries only the uuid — the definition is re-resolved inside
the run activity so a metric edited/deleted between sampling and execution is seen as it currently is."""
team_id: int
experiment_id: int
metric_uuid: str
metric_type: str # "funnel" | "mean" | "ratio"
@dataclasses.dataclass
class CanaryVariantStats:
sum: float
number_of_samples: int
@frozen
class CanaryRunSnapshot:
"""Per-variant aggregates from one execution of the metric query."""
label: str # "a" | "b" (forced precomputed) | "c" (forced direct scan)
query_id: str # client_query_id, for system.query_log forensics
is_precomputed: bool # exposures side
variants: dict[str, CanaryVariantStats]
# "precomputed" | "direct_scan" | "not_applicable". Defaulted so snapshots
# recorded before this field existed still decode during Temporal replay.
metric_events_path: str = "not_applicable"
@dataclasses.dataclass(frozen=False)
class CanaryMetricResult:
"""Verdict for one metric: outcome plus everything needed to investigate without re-running."""
target: CanaryMetricTarget
outcome: CanaryOutcome
stability_deviation: float | None = None # max relative deviation, run a vs b
correctness_deviation: float | None = None # max relative deviation, run b vs c
runs: list[CanaryRunSnapshot] = dataclasses.field(default_factory=list)
detail: str | None = None
@dataclasses.dataclass
class CanaryReportInputs:
results: list[CanaryMetricResult]
triggered_manually: bool = False
@dataclasses.dataclass
class RecalculationProgressUpdate:
"""Input for the start and finish lifecycle steps.
Start step: status=in_progress, total_metrics, metric_uuids, mark_started=True. The activity itself
stamps query_to and started_at under a first-write-wins guard.
Finish step: status=completed|failed, mark_completed=True. The activity stamps completed_at, also
first-write-wins.
Per-metric counters and error entries are not carried here — they're written by the calculate activity
in the same transaction as the result row.
"""
recalculation_id: str
status: str | None = None
total_metrics: int | None = None
metric_uuids: list[str] | None = None
mark_started: bool = False
mark_completed: bool = False
# Run-level outcome carried on the finish step so the activity can emit the
# 'experiment results refresh completed' analytics event with real counts.
succeeded_metrics: int | None = None
failed_metrics: int | None = None
ENROLLMENT_CENSUS_WORKFLOW_NAME = "experiment-precompute-enrollment-census"
@dataclasses.dataclass(frozen=True)
class ExperimentPrecomputeEnrollmentCensusInputs:
"""Input to the enrollment census workflow. Report-only: the census logs which teams
would qualify for precomputation enrollment; it never enrolls anyone."""
window_days: int = 14
SCHEDULED_RECALCULATION_WORKFLOW_NAME = "experiment-scheduled-recalculation-workflow"
@frozen
class ScheduledRecalculationStartResult:
"""Outcome of one experiment's start attempt, for the coordinator's summary counts."""
experiment_id: int
started: bool
recalculation_id: str | None = None
skip_reason: str | None = None