Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): add quantile statistic to the stats library (#127677)

Anders6f4745e1af0b
diff --git a/products/experiments/stats/bayesian/method.py b/products/experiments/stats/bayesian/method.py
index b0c5389985a9..3c58b6fc4cb4 100644
--- a/products/experiments/stats/bayesian/method.py
+++ b/products/experiments/stats/bayesian/method.py
@@ -10,7 +10,7 @@ from dataclasses import dataclass
from typing import Optional
from ..shared.enums import DifferenceType
-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError
+from ..shared.statistics import AnyStatistic, ProportionStatistic, SampleMeanStatistic, StatisticError
from .priors import GaussianPrior
from .tests import BayesianGaussianTest, BayesianMeanTest, BayesianProportionTest, BayesianResult, BayesianTest
@@ -90,7 +90,7 @@ class BayesianMethod:
Get or create test instance based on statistic type and configuration.
Args:
- stat_type: Type of statistic (SampleMeanStatistic or ProportionStatistic)
+ stat_type: Type of statistic; types without a dedicated test use BayesianGaussianTest
Returns:
Appropriate BayesianTest instance
@@ -109,8 +109,8 @@ class BayesianMethod:
def run_test(
self,
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
prior: GaussianPrior | None = None,
**kwargs,
) -> BayesianResult:
diff --git a/products/experiments/stats/bayesian/tests.py b/products/experiments/stats/bayesian/tests.py
index 0118e3a22f21..d24fe5e0f754 100644
--- a/products/experiments/stats/bayesian/tests.py
+++ b/products/experiments/stats/bayesian/tests.py
@@ -11,7 +11,7 @@ from dataclasses import dataclass
import numpy as np
from ..shared.enums import DifferenceType
-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError
+from ..shared.statistics import AnyStatistic, ProportionStatistic, SampleMeanStatistic, StatisticError
from .priors import GaussianPrior
from .utils import (
calculate_effect_size_and_variance,
@@ -87,8 +87,8 @@ class BayesianTest(ABC):
@abstractmethod
def run_test(
self,
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
prior: GaussianPrior,
difference_type: DifferenceType = DifferenceType.RELATIVE,
**kwargs,
@@ -126,8 +126,8 @@ class BayesianGaussianTest(BayesianTest):
def run_test(
self,
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
prior: GaussianPrior,
difference_type: DifferenceType = DifferenceType.RELATIVE,
**kwargs,
@@ -213,8 +213,8 @@ class BayesianProportionTest(BayesianTest):
def run_test(
self,
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
prior: GaussianPrior,
difference_type: DifferenceType = DifferenceType.RELATIVE,
**kwargs,
@@ -295,8 +295,8 @@ class BayesianMeanTest(BayesianTest):
def run_test(
self,
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
prior: GaussianPrior,
difference_type: DifferenceType = DifferenceType.RELATIVE,
**kwargs,
diff --git a/products/experiments/stats/bayesian/utils.py b/products/experiments/stats/bayesian/utils.py
index 854bc2156884..d6fb41bf55ea 100644
--- a/products/experiments/stats/bayesian/utils.py
+++ b/products/experiments/stats/bayesian/utils.py
@@ -8,14 +8,14 @@ posterior updates, credible intervals, and probability calculations.
from scipy.stats import norm
from ..shared.enums import DifferenceType
-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError
+from ..shared.statistics import AnyStatistic, StatisticError
from ..shared.utils import get_mean, get_sample_size, get_variance, validate_test_inputs
from .priors import GaussianPrior
def calculate_effect_size_and_variance(
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
difference_type: DifferenceType,
unadjusted_mean: float | None = None,
) -> tuple[float, float]:
@@ -202,8 +202,8 @@ def credible_interval(posterior_mean: float, posterior_std: float, alpha: float
def validate_inputs(
- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,
+ treatment_stat: AnyStatistic,
+ control_stat: AnyStatistic,
) -> None:
"""
Validate input statistics for Bayesian analysis.
diff --git a/products/experiments/stats/frequentist/tests.py b/products/experiments/stats/frequentist/tests.py
index f3f540a30e1d..1714dde8ddc2 100644
--- a/products/experiments/stats/frequentist/tests.py
+++ b/products/experiments/stats/frequentist/tests.py
@@ -15,6 +15,7 @@ from .utils import (
sequential_interval_halfwidth,
sequential_p_value,
sequential_rho,
+ validate_quantile_inputs,
validate_sample_sizes,
)
@@ -115,6 +116,7 @@ class TwoSidedTTest(StatisticalTest):
) -> TestResult:
"""Run two-sided t-test."""
validate_sample_sizes(treatment_stat, control_stat)
+ validate_quantile_inputs(treatment_stat, control_stat)
unadjusted_mean = kwargs.get("unadjusted_mean")
point_estimate = calculate_point_estimate(treatment_stat, control_stat, difference_type, unadjusted_mean)
@@ -181,6 +183,7 @@ class SequentialTwoSidedTTest(StatisticalTest):
**kwargs,
) -> TestResult:
validate_sample_sizes(treatment_stat, control_stat)
+ validate_quantile_inputs(treatment_stat, control_stat)
unadjusted_mean = kwargs.get("unadjusted_mean")
point_estimate = calculate_point_estimate(treatment_stat, control_stat, difference_type, unadjusted_mean)
diff --git a/products/experiments/stats/frequentist/utils.py b/products/experiments/stats/frequentist/utils.py
index 458d81e49ed7..2915e4202d7c 100644
--- a/products/experiments/stats/frequentist/utils.py
+++ b/products/experiments/stats/frequentist/utils.py
@@ -2,9 +2,9 @@ import numpy as np
from scipy import stats
from products.experiments.stats.shared.enums import DifferenceType
-from products.experiments.stats.shared.utils import get_mean, get_sample_size, get_variance
+from products.experiments.stats.shared.utils import get_mean, get_sample_size, get_variance, validate_test_inputs
-from ..shared.statistics import AnyStatistic, StatisticError
+from ..shared.statistics import AnyStatistic, QuantileStatistic, StatisticError
def calculate_point_estimate(
@@ -226,6 +226,22 @@ def validate_sample_sizes(treatment_stat: AnyStatistic, control_stat: AnyStatist
raise StatisticError(f"Control sample size ({control_n}) below minimum ({min_sample_size})")
+def validate_quantile_inputs(treatment_stat: AnyStatistic, control_stat: AnyStatistic) -> None:
+ """
+ Run the shared statistic validation when either arm is a QuantileStatistic.
+
+ The variance floor turns a degenerate quantile into a small positive variance, so the
+ frequentist tests would report a result instead of an error. Other statistic types
+ skip this check because the shared validation also rejects proportions near 0 or 1,
+ which the frequentist tests accept.
+
+ Raises:
+ StatisticError: If a quantile statistic fails validation
+ """
+ if isinstance(treatment_stat, QuantileStatistic) or isinstance(control_stat, QuantileStatistic):
+ validate_test_inputs(treatment_stat, control_stat)
+
+
def sequential_rho(alpha: float, sequential_tuning_parameter: float) -> float:
"""
Convert the sequential testing tuning parameter `N` into the mixing parameter `rho`.
diff --git a/products/experiments/stats/shared/statistics.py b/products/experiments/stats/shared/statistics.py
index fa3dba88ecd0..03f45f04a7ed 100644
--- a/products/experiments/stats/shared/statistics.py
+++ b/products/experiments/stats/shared/statistics.py
@@ -1,3 +1,4 @@
+import math
from dataclasses import dataclass
import numpy as np
@@ -150,5 +151,164 @@ class RatioStatistic:
return np.sqrt(self.variance / self.n)
+# Normal 0.975 quantile. The quantile variance uses it to turn an interval half-width
+# back into a standard error, so it cancels out and must not follow the experiment's alpha.
+Z_975 = 1.959963984540054
+
+# Reference sample sizes n* for the quantile interval. Query quantile levels must be
+# constants, but n is only known after the query runs, so the query returns bounds for
+# every grid size and choose_quantile_bounds picks one afterwards.
+QUANTILE_N_STAR_GRID = [100 * 2**k for k in range(20)]
+
+
+@dataclass(frozen=True, kw_only=True)
+class QuantileGridLevel:
+ """Probability levels of the outer quantile interval at one grid size n*."""
+
+ n_star: int
+ lower_level: float
+ upper_level: float
+
+
+@dataclass(frozen=True, kw_only=True)
+class QuantileBounds:
+ """Observed values Y_L and Y_U at the probability levels of one grid size n*."""
+
+ n_star: int
+ lower: float
+ upper: float
+
+
+def quantile_grid_levels(nu: float) -> list[QuantileGridLevel]:
+ """
+ Probability levels L and U of the outer interval around quantile level nu, one per grid size.
+
+ L and U are nu -/+ Z_975 * sqrt(nu(1-nu)/n*), clipped to (0, 1) and rounded to 10 decimals
+ so that the query text is stable.
+ """
+ levels = []
+ for n_star in QUANTILE_N_STAR_GRID:
+ se_p = math.sqrt(nu * (1 - nu) / n_star)
+ lower = round(max(nu - Z_975 * se_p, 1e-8), 10)
+ upper = round(min(nu + Z_975 * se_p, 1 - 1e-8), 10)
+ levels.append(QuantileGridLevel(n_star=n_star, lower_level=lower, upper_level=upper))
+ return levels
+
+
+def choose_quantile_bounds(n: int, bounds: list[QuantileBounds]) -> QuantileBounds | None:
+ """
+ Choose the bounds to use for a variant with n events.
+
+ bounds is in grid order. Picks the largest n* <= n, or the smallest grid size when n is
+ below it. Steps down to smaller n* while Y_L == Y_U (tied values).
+ Returns None when all eligible bounds collapse.
+ """
+ eligible = [b for b in bounds if b.n_star <= n] or [bounds[0]]
+ for candidate in reversed(eligible):
+ if candidate.upper > candidate.lower:
+ return candidate
+ return None
+
+
+@dataclass(frozen=True)
+class QuantileStatistic:
+ """
+ Statistics for event-level quantile metrics (e.g., p90 of page load time over all events).
+
+ The quantile is taken over all events of the variant, pooled across users. Users are the
+ randomization unit, so the variance is clustered by user with the delta method from
+ Deng, Knoblich and Lu (2018), "Applying the Delta Method in Metric Analytics".
+
+ n is the number of events, and variance is per event, so variance / n is the variance
+ of the quantile estimate.
+ """
+
+ nu: float # Quantile level, 0 < nu < 1
+ n: int # Number of qualifying events
+ n_clusters: int # Exposed users K, including users without events
+ quantile: float # Empirical nu-quantile of all event values
+ quantile_lower: float # Y_L at the chosen n*
+ quantile_upper: float # Y_U at the chosen n*
+ n_star: int # Chosen n* from QUANTILE_N_STAR_GRID
+ s_sum: float # Σ S_i, where S_i is the number of events of user i with value <= quantile
+ s_sum_squares: float # Σ S_i²
+ n_sum: float # Σ N_i, where N_i is the number of events of user i (equals n)
+ n_sum_squares: float # Σ N_i²
+ s_n_sum_of_products: float # Σ S_i·N_i
+
+ def __post_init__(self):
+ """Validate inputs."""
+ if not (0 < self.nu < 1):
+ raise InvalidStatisticError("Quantile level must be between 0 and 1")
+ if self.n <= 0:
+ raise InvalidStatisticError("Sample size must be positive")
+ if self.n_clusters <= 0:
+ raise InvalidStatisticError("Number of clusters must be positive")
+ if self.n_star <= 0:
+ raise InvalidStatisticError("Reference sample size must be positive")
+ if self.n_sum != self.n:
+ raise InvalidStatisticError(
+ "n_sum (Σ N_i, the total events over all users) must equal n (the number of events)"
+ )
+ if self.s_sum < 0 or self.s_sum > self.n_sum:
+ raise InvalidStatisticError("Events below the quantile must be between 0 and sample size")
+ if self.quantile_lower > self.quantile_upper:
+ raise InvalidStatisticError("Lower quantile bound cannot exceed upper quantile bound")
+
+ @property
+ def mean(self) -> float:
+ """Central value: the empirical quantile"""
+ return self.quantile
+
+ @property
+ def iid_variance(self) -> float:
+ """Per-event variance of the quantile if events were iid: (n*/n)(n-1)((Y_U - Y_L) / 2z)²"""
+ if self.n <= 1:
+ return 0.0
+ width = (self.quantile_upper - self.quantile_lower) / (2 * Z_975)
+ return (self.n_star / self.n) * (self.n - 1) * width**2
+
+ @property
+ def cdf_cluster_variance(self) -> float:
+ """Delta method variance of the empirical CDF at the quantile, Σ S_i / Σ N_i, over K users"""
+ k = self.n_clusters
+ if self.n <= 1 or self.nu == 0 or k <= 1 or self.n_sum <= 0:
+ return 0.0
+ mu_s = self.s_sum / k
+ mu_n = self.n_sum / k
+ var_s = (self.s_sum_squares / k - mu_s**2) * k / (k - 1)
+ var_n = (self.n_sum_squares / k - mu_n**2) * k / (k - 1)
+ cov_sn = (self.s_n_sum_of_products / k - mu_s * mu_n) * k / (k - 1)
+ return (var_s - 2 * mu_s * cov_sn / mu_n + mu_s**2 * var_n / mu_n**2) / (k * mu_n**2)
+
+ @property
+ def unfloored_variance(self) -> float:
+ """Clustered per-event variance: iid variance times the design effect of the CDF"""
+ cdf_iid_variance = self.nu * (1 - self.nu) / self.n if self.n > 0 else 0.0
+ if cdf_iid_variance <= 0:
+ return 0.0
+ return self.iid_variance * self.cdf_cluster_variance / cdf_iid_variance
+
+ @property
+ def variance(self) -> float:
+ """Clustered per-event variance, floored at 1e-5 when n >= 100"""
+ variance = self.unfloored_variance
+ return max(variance, 1e-5) if self.n >= 100 else variance
+
+ @property
+ def has_zero_variance(self) -> bool:
+ """True when the outer interval leaves [0, 1] or the variance is not positive"""
+ z2 = Z_975**2
+ if self.n <= z2 * self.nu / (1 - self.nu) or self.n <= z2 * (1 - self.nu) / self.nu:
+ return True
+ # The floor in variance turns collapsed bounds (Y_L == Y_U) into 1e-5, so read the value before it.
+ return self.unfloored_variance <= 0
+
+ @property
+ def standard_error(self) -> float:
+ """Standard error of the quantile: SE = √(variance/n)"""
+ return float(np.sqrt(self.variance / self.n))
+
+
# Type alias for any statistic type
-AnyStatistic = SampleMeanStatistic | ProportionStatistic | RatioStatistic
+AnyStatistic = SampleMeanStatistic | ProportionStatistic | RatioStatistic | QuantileStatistic
diff --git a/products/experiments/stats/shared/utils.py b/products/experiments/stats/shared/utils.py
index e661024faabb..eaa275cec8b2 100644
--- a/products/experiments/stats/shared/utils.py
+++ b/products/experiments/stats/shared/utils.py
@@ -7,7 +7,14 @@ by both frequentist and Bayesian methods.
import numpy as np
-from .statistics import AnyStatistic, ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError
+from .statistics import (
+ AnyStatistic,
+ ProportionStatistic,
+ QuantileStatistic,
+ RatioStatistic,
+ SampleMeanStatistic,
+ StatisticError,
+)
def get_mean(statistic: AnyStatistic) -> float:
@@ -18,6 +25,8 @@ def get_mean(statistic: AnyStatistic) -> float:
return statistic.proportion
elif isinstance(statistic, RatioStatistic):
return statistic.ratio
+ elif isinstance(statistic, QuantileStatistic):
+ return statistic.mean
else:
raise StatisticError(f"Unknown statistic type: {type(statistic)}")
@@ -64,6 +73,8 @@ def validate_statistic_inputs(statistic: AnyStatistic) -> list[str]:
errors.extend(_validate_sample_mean(statistic))
elif isinstance(statistic, ProportionStatistic):
errors.extend(_validate_proportion(statistic))
+ elif isinstance(statistic, QuantileStatistic):
+ errors.extend(_validate_quantile(statistic))
except Exception as e:
errors.append(f"Failed to validate statistic: {str(e)}")
@@ -110,6 +121,16 @@ def _validate_proportion(stat: ProportionStatistic) -> list[str]:
return errors
+def _validate_quantile(stat: QuantileStatistic) -> list[str]:
+ """Validate QuantileStatistic."""
+ errors = []
+
+ if stat.has_zero_variance:
+ errors.append("Quantile has zero variance - not enough events or too many tied values")
+
+ return errors
+
+
def validate_test_inputs(treatment_stat: AnyStatistic, control_stat: AnyStatistic) -> None:
"""
Comprehensive validation of test inputs.
diff --git a/products/experiments/stats/tests/test_bayesian.py b/products/experiments/stats/tests/test_bayesian.py
index ad15bd6b8e2a..e6cbd7a07103 100644
--- a/products/experiments/stats/tests/test_bayesian.py
+++ b/products/experiments/stats/tests/test_bayesian.py
@@ -11,6 +11,7 @@ import pytest
from unittest import TestCase
import numpy as np
+from parameterized import parameterized
from products.experiments.stats.bayesian.method import BayesianConfig, BayesianMethod
from products.experiments.stats.bayesian.priors import GaussianPrior
@@ -28,6 +29,11 @@ from products.experiments.stats.shared.statistics import (
SampleMeanStatistic,
StatisticError,
)
+from products.experiments.stats.tests.test_statistics import (
+ DEGENERATE_QUANTILE_CASES,
+ QUANTILE_CONTROL,
+ QUANTILE_TREATMENT,
+)
class TestGaussianPrior:
@@ -313,6 +319,40 @@ class TestBayesianMethod(TestCase):
self.assertAlmostEqual(credible_interval_result[0], expected_dict["credible_interval"][0], places=4)
self.assertAlmostEqual(credible_interval_result[1], expected_dict["credible_interval"][1], places=4)
+ @parameterized.expand(
+ [
+ (
+ "absolute",
+ DifferenceType.ABSOLUTE,
+ 1.415000000000001,
+ (0.08583772923349264, 2.744162270766509),
+ 0.9815350910332983,
+ ),
+ (
+ "relative",
+ DifferenceType.RELATIVE,
+ 0.12982842462611258,
+ (0.0031120207966113256, 0.2565448284556138),
+ 0.9776835938774414,
+ ),
+ ]
+ )
+ def test_gaussian_test_with_quantile_statistic(self, _name, difference_type, effect, interval, chance_to_win):
+ method = BayesianMethod(BayesianConfig(ci_level=0.95, difference_type=difference_type))
+ result = method.run_test(QUANTILE_TREATMENT, QUANTILE_CONTROL)
+
+ self.assertAlmostEqual(result.effect_size, effect, places=12)
+ self.assertAlmostEqual(result.credible_interval[0], interval[0], places=10)
+ self.assertAlmostEqual(result.credible_interval[1], interval[1], places=10)
+ self.assertAlmostEqual(result.chance_to_win, chance_to_win, places=10)
+
+ @parameterized.expand(DEGENERATE_QUANTILE_CASES)
+ def test_rejects_quantile_with_zero_variance(self, _name, treatment, control):
+ with self.assertRaisesRegex(
+ StatisticError, "Treatment statistic validation failed: Quantile has zero variance"
+ ):
+ BayesianMethod().run_test(treatment, control)
+
class TestConvenienceFunctions:
"""Tests for convenience functions."""
diff --git a/products/experiments/stats/tests/test_frequentist.py b/products/experiments/stats/tests/test_frequentist.py
index 37a4aa5f7587..dbcf87e68dda 100644
--- a/products/experiments/stats/tests/test_frequentist.py
+++ b/products/experiments/stats/tests/test_frequentist.py
@@ -20,6 +20,11 @@ from products.experiments.stats.shared.statistics import (
SampleMeanStatistic,
StatisticError,
)
+from products.experiments.stats.tests.test_statistics import (
+ DEGENERATE_QUANTILE_CASES,
+ QUANTILE_CONTROL,
+ QUANTILE_TREATMENT,
+)
def create_test_result_dict(result: Any) -> dict[str, Any]:
@@ -165,6 +170,47 @@ class TestTwoSidedTTest(TestCase):
self.assertAlmostEqual(uplift["mean"], expected_uplift["mean"], places=4)
self.assertAlmostEqual(uplift["stddev"], expected_uplift["stddev"], places=4)
+ @parameterized.expand(
+ [
+ (
+ "absolute",
+ DifferenceType.ABSOLUTE,
+ 1.415000000000001,
+ (0.08506091978298969, 2.744939080217012),
+ 0.0370517987954897,
+ ),
+ (
+ "relative",
+ DifferenceType.RELATIVE,
+ 0.12982842462611258,
+ (0.003037963247587039, 0.2566188860046381),
+ 0.044762375592802384,
+ ),
+ ]
+ )
+ def test_two_sided_ttest_with_quantile_statistic(self, _name, difference_type, expected, ci, p_value):
+ method = FrequentistMethod(FrequentistConfig(alpha=0.05, difference_type=difference_type))
+ result = method.run_test(QUANTILE_TREATMENT, QUANTILE_CONTROL)
+
+ self.assertAlmostEqual(result.point_estimate, expected, places=12)
+ self.assertAlmostEqual(result.confidence_interval[0], ci[0], places=10)
+ self.assertAlmostEqual(result.confidence_interval[1], ci[1], places=10)
+ self.assertAlmostEqual(result.p_value, p_value, places=9)
+
+ @parameterized.expand(
+ [
+ (f"{name}_{mode}", treatment, control, sequential)
+ for name, treatment, control in DEGENERATE_QUANTILE_CASES
+ for mode, sequential in [("fixed_horizon", False), ("sequential", True)]
+ ]
+ )
+ def test_rejects_quantile_with_zero_variance(self, _name, treatment, control, sequential):
+ method = FrequentistMethod(FrequentistConfig(sequential_testing_enabled=sequential))
+ with self.assertRaisesRegex(
+ StatisticError, "Treatment statistic validation failed: Quantile has zero variance"
+ ):
+ method.run_test(treatment, control)
+
class TestSequentialMath(TestCase):
"""Unit tests for the sequential testing primitives (rho, half-width, p-value)."""
diff --git a/products/experiments/stats/tests/test_statistics.py b/products/experiments/stats/tests/test_statistics.py
index fc1e135457dc..25092f2bd097 100644
--- a/products/experiments/stats/tests/test_statistics.py
+++ b/products/experiments/stats/tests/test_statistics.py
@@ -1,8 +1,93 @@
+from typing import Any
+
import pytest
+from unittest import TestCase
import numpy as np
-
-from ..shared.statistics import InvalidStatisticError, ProportionStatistic, RatioStatistic, SampleMeanStatistic
+from parameterized import parameterized
+
+from ..shared.statistics import (
+ QUANTILE_N_STAR_GRID,
+ InvalidStatisticError,
+ ProportionStatistic,
+ QuantileBounds,
+ QuantileGridLevel,
+ QuantileStatistic,
+ RatioStatistic,
+ SampleMeanStatistic,
+ choose_quantile_bounds,
+ quantile_grid_levels,
+)
+
+# Sufficient statistics of a seeded synthetic dataset: 400 users per arm, Poisson(3) events
+# per user (some users have none), lognormal values with a per-user random effect, nu = 0.9.
+QUANTILE_CONTROL = QuantileStatistic(
+ nu=0.9,
+ n=1263,
+ n_clusters=400,
+ quantile=10.899,
+ quantile_lower=10.34,
+ quantile_upper=11.633,
+ n_star=800,
+ s_sum=1137,
+ s_sum_squares=4443,
+ n_sum=1263,
+ n_sum_squares=5193,
+ s_n_sum_of_products=4684,
+)
+QUANTILE_TREATMENT = QuantileStatistic(
+ nu=0.9,
+ n=1217,
+ n_clusters=400,
+ quantile=12.314,
+ quantile_lower=11.178,
+ quantile_upper=13.101,
+ n_star=800,
+ s_sum=1096,
+ s_sum_squares=4138,
+ n_sum=1217,
+ n_sum_squares=4793,
+ s_n_sum_of_products=4330,
+)
+
+
+def _iid_quantile(nu: float, n: int, lower: float, upper: float, n_clusters: int | None = None) -> QuantileStatistic:
+ # One event per user, so the design effect is K / (K - 1).
+ k = n if n_clusters is None else n_clusters
+ return QuantileStatistic(
+ nu=nu,
+ n=n,
+ n_clusters=k,
+ quantile=(lower + upper) / 2,
+ quantile_lower=lower,
+ quantile_upper=upper,
+ n_star=max([n_star for n_star in QUANTILE_N_STAR_GRID if n_star <= n], default=QUANTILE_N_STAR_GRID[0]),
+ s_sum=nu * n,
+ s_sum_squares=nu * n,
+ n_sum=n,
+ n_sum_squares=n,
+ s_n_sum_of_products=nu * n,
+ )
+
+
+# (name, treatment, control) pairs in which the treatment quantile has no usable variance.
+DEGENERATE_QUANTILE_CASES = [
+ (
+ "collapsed_bounds",
+ _iid_quantile(nu=0.6, n=700, lower=4.0, upper=4.0),
+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),
+ ),
+ (
+ "tail_too_small_for_quantile",
+ _iid_quantile(nu=0.98, n=150, lower=3.9, upper=4.1),
+ _iid_quantile(nu=0.98, n=6_000, lower=3.9, upper=4.1),
+ ),
+ (
+ "single_cluster",
+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1, n_clusters=1),
+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),
+ ),
+]
class TestSampleMeanStatistic:
@@ -176,3 +261,93 @@ class TestStatisticIntegration:
assert abs(sample_mean.standard_error - expected_se_mean) < 1e-10
assert abs(proportion.standard_error - expected_se_prop) < 1e-10
+
+
+class TestQuantileStatistic(TestCase):
+ @parameterized.expand(
+ [
+ ("control", QUANTILE_CONTROL, 86.97348405307501, 0.00013708395792512163, 167.31479431374606),
+ ("treatment", QUANTILE_TREATMENT, 192.3691290860306, 0.0001531846058978492, 398.4726987554284),
+ ]
+ )
+ def test_clustered_variance(self, _name, stat, iid_variance, cdf_cluster_variance, variance):
+ assert stat.mean == stat.quantile
+ assert stat.iid_variance == pytest.approx(iid_variance, rel=1e-12)
+ assert stat.cdf_cluster_variance == pytest.approx(cdf_cluster_variance, rel=1e-12)
+ assert stat.variance == pytest.approx(variance, rel=1e-12)
+ assert stat.standard_error == pytest.approx(np.sqrt(variance / stat.n), rel=1e-12)
+ assert not stat.has_zero_variance
+
+ @parameterized.expand(
+ [
+ ("floored_at_n_100_or_more", 400, 1e-5),
+ ("not_floored_below_n_100", 50, 6.507944290644809e-08),
+ ]
+ )
+ def test_variance_floor(self, _name, n, expected_variance):
+ stat = _iid_quantile(nu=0.5, n=n, lower=10.0, upper=10.0001)
+ assert stat.variance == pytest.approx(expected_variance, rel=1e-9)
+
+ @parameterized.expand(
+ [
+ ("collapsed_bounds_large_n", 0.75, 50_000_000, 8.0, 8.0, None, True),
+ ("collapsed_bounds_small_n", 0.75, 2_000, 8.0, 8.0, None, True),
+ ("open_bounds_large_n", 0.75, 50_000_000, 7.98, 8.02, None, False),
+ ("interval_leaves_unit_range", 0.99, 300, 7.98, 8.02, None, True),
+ ("single_cluster", 0.75, 2_000, 7.98, 8.02, 1, True),
+ ]
+ )
+ def test_has_zero_variance(self, _name, nu, n, lower, upper, n_clusters, expected):
+ stat = _iid_quantile(nu=nu, n=n, lower=lower, upper=upper, n_clusters=n_clusters)
+ assert stat.has_zero_variance is expected
+
+ @parameterized.expand(
+ [
+ ("nu_out_of_range", {"nu": 1.0}),
+ ("no_events", {"n": 0, "n_sum": 0}),
+ ("no_clusters", {"n_clusters": 0}),
+ ("n_sum_differs_from_n", {"n_sum": 1262}),
+ ("s_sum_above_n_sum", {"s_sum": 1264}),
+ ("lower_above_upper", {"quantile_lower": 12.0}),
+ ]
+ )
+ def test_validation_errors(self, _name, overrides: dict[str, Any]):
+ fields = {name: getattr(QUANTILE_CONTROL, name) for name in QUANTILE_CONTROL.__dataclass_fields__}
+ with pytest.raises(InvalidStatisticError):
+ QuantileStatistic(**{**fields, **overrides})
+
+
+class TestQuantileGridLevels(TestCase):
+ @parameterized.expand(
+ [
+ ("nu_75_smallest_grid", 0.75, 0, (100, 0.6651310699, 0.8348689301)),
+ ("nu_75_n_star_1600", 0.75, 4, (1600, 0.7287827675, 0.7712172325)),
+ ("upper_clamped_below_one", 0.99, 0, (100, 0.9704986046, 0.99999999)),
+ ]
+ )
+ def test_levels(self, _name, nu, index, expected):
+ levels = quantile_grid_levels(nu)
+ assert [level.n_star for level in levels] == QUANTILE_N_STAR_GRID
+ n_star, lower_level, upper_level = expected
+ assert levels[index] == QuantileGridLevel(n_star=n_star, lower_level=lower_level, upper_level=upper_level)
+
+
+class TestChooseQuantileBounds(TestCase):
+ BOUNDS = [(100, 1.0, 3.0), (200, 1.5, 2.5), (400, 1.8, 2.2), (800, 2.0, 2.0)]
+
+ @staticmethod
+ def _bounds(n_star: int, lower: float, upper: float) -> QuantileBounds:
+ return QuantileBounds(n_star=n_star, lower=lower, upper=upper)
+
+ @parameterized.expand(
+ [
+ ("largest_n_star_below_n", 300, BOUNDS, (200, 1.5, 2.5)),
+ ("n_star_equal_to_n", 400, BOUNDS, (400, 1.8, 2.2)),
+ ("smallest_grid_size_when_n_below_it", 40, BOUNDS, (100, 1.0, 3.0)),
+ ("steps_down_when_bounds_collapse", 1000, BOUNDS, (400, 1.8, 2.2)),
+ ("none_when_all_bounds_collapse", 1000, [(100, 2.0, 2.0), (200, 2.0, 2.0)], None),
+ ]
+ )
+ def test_choose(self, _name, n, bounds, expected):
+ chosen = choose_quantile_bounds(n, [self._bounds(*b) for b in bounds])
+ assert chosen == (self._bounds(*expected) if expected is not None else None)