:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import math
from dataclasses import dataclass
import numpy as np
class StatisticError(ValueError):
"""Base exception for statistic-related errors."""
pass
class InvalidStatisticError(StatisticError):
"""Raised when statistic inputs are invalid."""
pass
@dataclass
class SampleMeanStatistic:
"""
Statistics for continuous metrics (e.g., revenue, session duration).
For a sample of continuous values, this stores sufficient statistics
to compute the sample mean and variance.
"""
n: int # Sample size (number of observations)
sum: float # Sum of all observations
sum_squares: float # Sum of squared observations
def __post_init__(self):
"""Validate inputs and check mathematical constraints."""
if self.n <= 0:
raise InvalidStatisticError("Sample size must be positive")
if self.n == 1 and self.sum_squares < self.sum**2:
raise InvalidStatisticError("sum_squares must be >= sum^2 for single observation")
if self.n > 1 and self.sum_squares < self.sum**2 / self.n:
raise InvalidStatisticError("sum_squares incompatible with sum and n")
@property
def mean(self) -> float:
"""Sample mean: μ = sum / n"""
return self.sum / self.n
@property
def variance(self) -> float:
"""Sample variance: s² = (sum_squares - sum²/n) / (n-1)"""
if self.n == 1:
return 0.0
return (self.sum_squares - self.sum**2 / self.n) / (self.n - 1)
@property
def standard_error(self) -> float:
"""Standard error of the mean: SE = √(s²/n)"""
return np.sqrt(self.variance / self.n)
@dataclass
class ProportionStatistic:
"""
Statistics for binary/conversion metrics (e.g., conversion rate, CTR).
For a sample of binary outcomes (success/failure), this stores
the number of trials and successes.
"""
n: int # Sample size (number of users/units exposed)
sum: int # Number of successes/conversions
def __post_init__(self):
"""Validate inputs."""
if self.n <= 0:
raise InvalidStatisticError("Sample size must be positive")
if self.sum < 0:
raise InvalidStatisticError("Number of successes cannot be negative")
if self.sum > self.n:
raise InvalidStatisticError("Number of successes cannot exceed sample size")
@property
def proportion(self) -> float:
"""Sample proportion: p = sum / n"""
return self.sum / self.n
@property
def variance(self) -> float:
"""Binomial variance: Var(p) = p(1-p)"""
p = self.proportion
return p * (1 - p)
@property
def standard_error(self) -> float:
"""Standard error of proportion: SE = √(p(1-p)/n)"""
return np.sqrt(self.variance / self.n)
@dataclass
class RatioStatistic:
"""
Statistics for ratio metrics (e.g., revenue per order, clicks per session).
Uses the delta method for variance approximation of ratios.
"""
n: int # Sample size
m_statistic: SampleMeanStatistic | ProportionStatistic # Numerator statistic
d_statistic: SampleMeanStatistic | ProportionStatistic # Denominator statistic
m_d_sum_of_products: float # Sum of products between numerator and denominator
def __post_init__(self):
"""Validate inputs."""
if self.n <= 0:
raise InvalidStatisticError("Sample size must be positive")
if self.m_statistic.n != self.n or self.d_statistic.n != self.n:
raise InvalidStatisticError("All statistics must have same sample size")
d_mean = self.d_statistic.mean if hasattr(self.d_statistic, "mean") else self.d_statistic.proportion
if abs(d_mean) < 1e-10:
raise InvalidStatisticError("Denominator mean cannot be zero for ratio calculation")
@property
def ratio(self) -> float:
"""Ratio estimate: R = Σm / Σd"""
m_sum = self.m_statistic.sum
d_sum = self.d_statistic.sum
return m_sum / d_sum
@property
def covariance(self) -> float:
"""Sample covariance: Cov(M,D) = (sum_products - sum_m × sum_d / n) / (n-1)"""
if self.n == 1:
return 0.0
m_sum = self.m_statistic.sum
d_sum = self.d_statistic.sum
return (self.m_d_sum_of_products - m_sum * d_sum / self.n) / (self.n - 1)
@property
def variance(self) -> float:
"""Delta method variance for ratio R = M/D"""
m_mean = self.m_statistic.mean if hasattr(self.m_statistic, "mean") else self.m_statistic.proportion
d_mean = self.d_statistic.mean if hasattr(self.d_statistic, "mean") else self.d_statistic.proportion
m_var = self.m_statistic.variance
d_var = self.d_statistic.variance
cov = self.covariance
# Delta method: Var(R) ≈ Var(M)/D² + M²Var(D)/D⁴ - 2M*Cov(M,D)/D³
return m_var / d_mean**2 + m_mean**2 * d_var / d_mean**4 - 2 * m_mean * cov / d_mean**3
@property
def standard_error(self) -> float:
"""Standard error of the ratio: SE = √(Var(R)/n)"""
return np.sqrt(self.variance / self.n)
# Normal 0.975 quantile. The quantile variance uses it to turn an interval half-width
# back into a standard error, so it cancels out and must not follow the experiment's alpha.
Z_975 = 1.959963984540054
# Reference sample sizes n* for the quantile interval. Query quantile levels must be
# constants, but n is only known after the query runs, so the query returns bounds for
# every grid size and choose_quantile_bounds picks one afterwards.
QUANTILE_N_STAR_GRID = [100 * 2**k for k in range(20)]
@dataclass(frozen=True, kw_only=True)
class QuantileGridLevel:
"""Probability levels of the outer quantile interval at one grid size n*."""
n_star: int
lower_level: float
upper_level: float
@dataclass(frozen=True, kw_only=True)
class QuantileBounds:
"""Observed values Y_L and Y_U at the probability levels of one grid size n*."""
n_star: int
lower: float
upper: float
def quantile_grid_levels(nu: float) -> list[QuantileGridLevel]:
"""
Probability levels L and U of the outer interval around quantile level nu, one per grid size.
L and U are nu -/+ Z_975 * sqrt(nu(1-nu)/n*), clipped to (0, 1) and rounded to 10 decimals
so that the query text is stable.
"""
levels = []
for n_star in QUANTILE_N_STAR_GRID:
se_p = math.sqrt(nu * (1 - nu) / n_star)
lower = round(max(nu - Z_975 * se_p, 1e-8), 10)
upper = round(min(nu + Z_975 * se_p, 1 - 1e-8), 10)
levels.append(QuantileGridLevel(n_star=n_star, lower_level=lower, upper_level=upper))
return levels
def choose_quantile_bounds(n: int, bounds: list[QuantileBounds]) -> QuantileBounds | None:
"""
Choose the bounds to use for a variant with n events.
bounds is in grid order. Picks the largest n* <= n, or the smallest grid size when n is
below it. Steps down to smaller n* while Y_L == Y_U (tied values).
Returns None when all eligible bounds collapse.
"""
eligible = [b for b in bounds if b.n_star <= n] or [bounds[0]]
for candidate in reversed(eligible):
if candidate.upper > candidate.lower:
return candidate
return None
@dataclass(frozen=True)
class QuantileStatistic:
"""
Statistics for event-level quantile metrics (e.g., p90 of page load time over all events).
The quantile is taken over all events of the variant, pooled across users. Users are the
randomization unit, so the variance is clustered by user with the delta method from
Deng, Knoblich and Lu (2018), "Applying the Delta Method in Metric Analytics".
n is the number of events, and variance is per event, so variance / n is the variance
of the quantile estimate.
"""
nu: float # Quantile level, 0 < nu < 1
n: int # Number of qualifying events
n_clusters: int # Exposed users K, including users without events
quantile: float # Empirical nu-quantile of all event values
quantile_lower: float # Y_L at the chosen n*
quantile_upper: float # Y_U at the chosen n*
n_star: int # Chosen n* from QUANTILE_N_STAR_GRID
s_sum: float # Σ S_i, where S_i is the number of events of user i with value <= quantile
s_sum_squares: float # Σ S_i²
n_sum: float # Σ N_i, where N_i is the number of events of user i (equals n)
n_sum_squares: float # Σ N_i²
s_n_sum_of_products: float # Σ S_i·N_i
def __post_init__(self):
"""Validate inputs."""
if not (0 < self.nu < 1):
raise InvalidStatisticError("Quantile level must be between 0 and 1")
if self.n <= 0:
raise InvalidStatisticError("Sample size must be positive")
if self.n_clusters <= 0:
raise InvalidStatisticError("Number of clusters must be positive")
if self.n_star <= 0:
raise InvalidStatisticError("Reference sample size must be positive")
if self.n_sum != self.n:
raise InvalidStatisticError(
"n_sum (Σ N_i, the total events over all users) must equal n (the number of events)"
)
if self.s_sum < 0 or self.s_sum > self.n_sum:
raise InvalidStatisticError("Events below the quantile must be between 0 and sample size")
if self.quantile_lower > self.quantile_upper:
raise InvalidStatisticError("Lower quantile bound cannot exceed upper quantile bound")
@property
def mean(self) -> float:
"""Central value: the empirical quantile"""
return self.quantile
@property
def iid_variance(self) -> float:
"""Per-event variance of the quantile if events were iid: (n*/n)(n-1)((Y_U - Y_L) / 2z)²"""
if self.n <= 1:
return 0.0
width = (self.quantile_upper - self.quantile_lower) / (2 * Z_975)
return (self.n_star / self.n) * (self.n - 1) * width**2
@property
def cdf_cluster_variance(self) -> float:
"""Delta method variance of the empirical CDF at the quantile, Σ S_i / Σ N_i, over K users"""
k = self.n_clusters
if self.n <= 1 or self.nu == 0 or k <= 1 or self.n_sum <= 0:
return 0.0
mu_s = self.s_sum / k
mu_n = self.n_sum / k
var_s = (self.s_sum_squares / k - mu_s**2) * k / (k - 1)
var_n = (self.n_sum_squares / k - mu_n**2) * k / (k - 1)
cov_sn = (self.s_n_sum_of_products / k - mu_s * mu_n) * k / (k - 1)
return (var_s - 2 * mu_s * cov_sn / mu_n + mu_s**2 * var_n / mu_n**2) / (k * mu_n**2)
@property
def unfloored_variance(self) -> float:
"""Clustered per-event variance: iid variance times the design effect of the CDF"""
cdf_iid_variance = self.nu * (1 - self.nu) / self.n if self.n > 0 else 0.0
if cdf_iid_variance <= 0:
return 0.0
return self.iid_variance * self.cdf_cluster_variance / cdf_iid_variance
@property
def variance(self) -> float:
"""Clustered per-event variance, floored at 1e-5 when n >= 100"""
variance = self.unfloored_variance
return max(variance, 1e-5) if self.n >= 100 else variance
@property
def has_zero_variance(self) -> bool:
"""True when the outer interval leaves [0, 1] or the variance is not positive"""
z2 = Z_975**2
if self.n <= z2 * self.nu / (1 - self.nu) or self.n <= z2 * (1 - self.nu) / self.nu:
return True
# The floor in variance turns collapsed bounds (Y_L == Y_U) into 1e-5, so read the value before it.
return self.unfloored_variance <= 0
@property
def standard_error(self) -> float:
"""Standard error of the quantile: SE = √(variance/n)"""
return float(np.sqrt(self.variance / self.n))
# Type alias for any statistic type
AnyStatistic = SampleMeanStatistic | ProportionStatistic | RatioStatistic | QuantileStatistic