:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): add quantile statistic to the stats library (#127677)
Anders6f4745e1af0b
diff --git a/products/experiments/stats/bayesian/method.py b/products/experiments/stats/bayesian/method.pyindex b0c5389985a9..3c58b6fc4cb4 100644--- a/products/experiments/stats/bayesian/method.py+++ b/products/experiments/stats/bayesian/method.py@@ -10,7 +10,7 @@ from dataclasses import dataclassfrom typing import Optionalfrom ..shared.enums import DifferenceType-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError+from ..shared.statistics import AnyStatistic, ProportionStatistic, SampleMeanStatistic, StatisticErrorfrom .priors import GaussianPriorfrom .tests import BayesianGaussianTest, BayesianMeanTest, BayesianProportionTest, BayesianResult, BayesianTest@@ -90,7 +90,7 @@ class BayesianMethod:Get or create test instance based on statistic type and configuration.Args:- stat_type: Type of statistic (SampleMeanStatistic or ProportionStatistic)+ stat_type: Type of statistic; types without a dedicated test use BayesianGaussianTestReturns:Appropriate BayesianTest instance@@ -109,8 +109,8 @@ class BayesianMethod:def run_test(self,- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,prior: GaussianPrior | None = None,**kwargs,) -> BayesianResult:diff --git a/products/experiments/stats/bayesian/tests.py b/products/experiments/stats/bayesian/tests.pyindex 0118e3a22f21..d24fe5e0f754 100644--- a/products/experiments/stats/bayesian/tests.py+++ b/products/experiments/stats/bayesian/tests.py@@ -11,7 +11,7 @@ from dataclasses import dataclassimport numpy as npfrom ..shared.enums import DifferenceType-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError+from ..shared.statistics import AnyStatistic, ProportionStatistic, SampleMeanStatistic, StatisticErrorfrom .priors import GaussianPriorfrom .utils import (calculate_effect_size_and_variance,@@ -87,8 +87,8 @@ class BayesianTest(ABC):@abstractmethoddef run_test(self,- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,prior: GaussianPrior,difference_type: DifferenceType = DifferenceType.RELATIVE,**kwargs,@@ -126,8 +126,8 @@ class BayesianGaussianTest(BayesianTest):def run_test(self,- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,prior: GaussianPrior,difference_type: DifferenceType = DifferenceType.RELATIVE,**kwargs,@@ -213,8 +213,8 @@ class BayesianProportionTest(BayesianTest):def run_test(self,- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,prior: GaussianPrior,difference_type: DifferenceType = DifferenceType.RELATIVE,**kwargs,@@ -295,8 +295,8 @@ class BayesianMeanTest(BayesianTest):def run_test(self,- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,prior: GaussianPrior,difference_type: DifferenceType = DifferenceType.RELATIVE,**kwargs,diff --git a/products/experiments/stats/bayesian/utils.py b/products/experiments/stats/bayesian/utils.pyindex 854bc2156884..d6fb41bf55ea 100644--- a/products/experiments/stats/bayesian/utils.py+++ b/products/experiments/stats/bayesian/utils.py@@ -8,14 +8,14 @@ posterior updates, credible intervals, and probability calculations.from scipy.stats import normfrom ..shared.enums import DifferenceType-from ..shared.statistics import ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError+from ..shared.statistics import AnyStatistic, StatisticErrorfrom ..shared.utils import get_mean, get_sample_size, get_variance, validate_test_inputsfrom .priors import GaussianPriordef calculate_effect_size_and_variance(- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,difference_type: DifferenceType,unadjusted_mean: float | None = None,) -> tuple[float, float]:@@ -202,8 +202,8 @@ def credible_interval(posterior_mean: float, posterior_std: float, alpha: floatdef validate_inputs(- treatment_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,- control_stat: SampleMeanStatistic | ProportionStatistic | RatioStatistic,+ treatment_stat: AnyStatistic,+ control_stat: AnyStatistic,) -> None:"""Validate input statistics for Bayesian analysis.diff --git a/products/experiments/stats/frequentist/tests.py b/products/experiments/stats/frequentist/tests.pyindex f3f540a30e1d..1714dde8ddc2 100644--- a/products/experiments/stats/frequentist/tests.py+++ b/products/experiments/stats/frequentist/tests.py@@ -15,6 +15,7 @@ from .utils import (sequential_interval_halfwidth,sequential_p_value,sequential_rho,+ validate_quantile_inputs,validate_sample_sizes,)@@ -115,6 +116,7 @@ class TwoSidedTTest(StatisticalTest):) -> TestResult:"""Run two-sided t-test."""validate_sample_sizes(treatment_stat, control_stat)+ validate_quantile_inputs(treatment_stat, control_stat)unadjusted_mean = kwargs.get("unadjusted_mean")point_estimate = calculate_point_estimate(treatment_stat, control_stat, difference_type, unadjusted_mean)@@ -181,6 +183,7 @@ class SequentialTwoSidedTTest(StatisticalTest):**kwargs,) -> TestResult:validate_sample_sizes(treatment_stat, control_stat)+ validate_quantile_inputs(treatment_stat, control_stat)unadjusted_mean = kwargs.get("unadjusted_mean")point_estimate = calculate_point_estimate(treatment_stat, control_stat, difference_type, unadjusted_mean)diff --git a/products/experiments/stats/frequentist/utils.py b/products/experiments/stats/frequentist/utils.pyindex 458d81e49ed7..2915e4202d7c 100644--- a/products/experiments/stats/frequentist/utils.py+++ b/products/experiments/stats/frequentist/utils.py@@ -2,9 +2,9 @@ import numpy as npfrom scipy import statsfrom products.experiments.stats.shared.enums import DifferenceType-from products.experiments.stats.shared.utils import get_mean, get_sample_size, get_variance+from products.experiments.stats.shared.utils import get_mean, get_sample_size, get_variance, validate_test_inputs-from ..shared.statistics import AnyStatistic, StatisticError+from ..shared.statistics import AnyStatistic, QuantileStatistic, StatisticErrordef calculate_point_estimate(@@ -226,6 +226,22 @@ def validate_sample_sizes(treatment_stat: AnyStatistic, control_stat: AnyStatistraise StatisticError(f"Control sample size ({control_n}) below minimum ({min_sample_size})")+def validate_quantile_inputs(treatment_stat: AnyStatistic, control_stat: AnyStatistic) -> None:+ """+ Run the shared statistic validation when either arm is a QuantileStatistic.++ The variance floor turns a degenerate quantile into a small positive variance, so the+ frequentist tests would report a result instead of an error. Other statistic types+ skip this check because the shared validation also rejects proportions near 0 or 1,+ which the frequentist tests accept.++ Raises:+ StatisticError: If a quantile statistic fails validation+ """+ if isinstance(treatment_stat, QuantileStatistic) or isinstance(control_stat, QuantileStatistic):+ validate_test_inputs(treatment_stat, control_stat)++def sequential_rho(alpha: float, sequential_tuning_parameter: float) -> float:"""Convert the sequential testing tuning parameter `N` into the mixing parameter `rho`.diff --git a/products/experiments/stats/shared/statistics.py b/products/experiments/stats/shared/statistics.pyindex fa3dba88ecd0..03f45f04a7ed 100644--- a/products/experiments/stats/shared/statistics.py+++ b/products/experiments/stats/shared/statistics.py@@ -1,3 +1,4 @@+import mathfrom dataclasses import dataclassimport numpy as np@@ -150,5 +151,164 @@ class RatioStatistic:return np.sqrt(self.variance / self.n)+# Normal 0.975 quantile. The quantile variance uses it to turn an interval half-width+# back into a standard error, so it cancels out and must not follow the experiment's alpha.+Z_975 = 1.959963984540054++# Reference sample sizes n* for the quantile interval. Query quantile levels must be+# constants, but n is only known after the query runs, so the query returns bounds for+# every grid size and choose_quantile_bounds picks one afterwards.+QUANTILE_N_STAR_GRID = [100 * 2**k for k in range(20)]+++@dataclass(frozen=True, kw_only=True)+class QuantileGridLevel:+ """Probability levels of the outer quantile interval at one grid size n*."""++ n_star: int+ lower_level: float+ upper_level: float+++@dataclass(frozen=True, kw_only=True)+class QuantileBounds:+ """Observed values Y_L and Y_U at the probability levels of one grid size n*."""++ n_star: int+ lower: float+ upper: float+++def quantile_grid_levels(nu: float) -> list[QuantileGridLevel]:+ """+ Probability levels L and U of the outer interval around quantile level nu, one per grid size.++ L and U are nu -/+ Z_975 * sqrt(nu(1-nu)/n*), clipped to (0, 1) and rounded to 10 decimals+ so that the query text is stable.+ """+ levels = []+ for n_star in QUANTILE_N_STAR_GRID:+ se_p = math.sqrt(nu * (1 - nu) / n_star)+ lower = round(max(nu - Z_975 * se_p, 1e-8), 10)+ upper = round(min(nu + Z_975 * se_p, 1 - 1e-8), 10)+ levels.append(QuantileGridLevel(n_star=n_star, lower_level=lower, upper_level=upper))+ return levels+++def choose_quantile_bounds(n: int, bounds: list[QuantileBounds]) -> QuantileBounds | None:+ """+ Choose the bounds to use for a variant with n events.++ bounds is in grid order. Picks the largest n* <= n, or the smallest grid size when n is+ below it. Steps down to smaller n* while Y_L == Y_U (tied values).+ Returns None when all eligible bounds collapse.+ """+ eligible = [b for b in bounds if b.n_star <= n] or [bounds[0]]+ for candidate in reversed(eligible):+ if candidate.upper > candidate.lower:+ return candidate+ return None+++@dataclass(frozen=True)+class QuantileStatistic:+ """+ Statistics for event-level quantile metrics (e.g., p90 of page load time over all events).++ The quantile is taken over all events of the variant, pooled across users. Users are the+ randomization unit, so the variance is clustered by user with the delta method from+ Deng, Knoblich and Lu (2018), "Applying the Delta Method in Metric Analytics".++ n is the number of events, and variance is per event, so variance / n is the variance+ of the quantile estimate.+ """++ nu: float # Quantile level, 0 < nu < 1+ n: int # Number of qualifying events+ n_clusters: int # Exposed users K, including users without events+ quantile: float # Empirical nu-quantile of all event values+ quantile_lower: float # Y_L at the chosen n*+ quantile_upper: float # Y_U at the chosen n*+ n_star: int # Chosen n* from QUANTILE_N_STAR_GRID+ s_sum: float # Σ S_i, where S_i is the number of events of user i with value <= quantile+ s_sum_squares: float # Σ S_i²+ n_sum: float # Σ N_i, where N_i is the number of events of user i (equals n)+ n_sum_squares: float # Σ N_i²+ s_n_sum_of_products: float # Σ S_i·N_i++ def __post_init__(self):+ """Validate inputs."""+ if not (0 < self.nu < 1):+ raise InvalidStatisticError("Quantile level must be between 0 and 1")+ if self.n <= 0:+ raise InvalidStatisticError("Sample size must be positive")+ if self.n_clusters <= 0:+ raise InvalidStatisticError("Number of clusters must be positive")+ if self.n_star <= 0:+ raise InvalidStatisticError("Reference sample size must be positive")+ if self.n_sum != self.n:+ raise InvalidStatisticError(+ "n_sum (Σ N_i, the total events over all users) must equal n (the number of events)"+ )+ if self.s_sum < 0 or self.s_sum > self.n_sum:+ raise InvalidStatisticError("Events below the quantile must be between 0 and sample size")+ if self.quantile_lower > self.quantile_upper:+ raise InvalidStatisticError("Lower quantile bound cannot exceed upper quantile bound")++ @property+ def mean(self) -> float:+ """Central value: the empirical quantile"""+ return self.quantile++ @property+ def iid_variance(self) -> float:+ """Per-event variance of the quantile if events were iid: (n*/n)(n-1)((Y_U - Y_L) / 2z)²"""+ if self.n <= 1:+ return 0.0+ width = (self.quantile_upper - self.quantile_lower) / (2 * Z_975)+ return (self.n_star / self.n) * (self.n - 1) * width**2++ @property+ def cdf_cluster_variance(self) -> float:+ """Delta method variance of the empirical CDF at the quantile, Σ S_i / Σ N_i, over K users"""+ k = self.n_clusters+ if self.n <= 1 or self.nu == 0 or k <= 1 or self.n_sum <= 0:+ return 0.0+ mu_s = self.s_sum / k+ mu_n = self.n_sum / k+ var_s = (self.s_sum_squares / k - mu_s**2) * k / (k - 1)+ var_n = (self.n_sum_squares / k - mu_n**2) * k / (k - 1)+ cov_sn = (self.s_n_sum_of_products / k - mu_s * mu_n) * k / (k - 1)+ return (var_s - 2 * mu_s * cov_sn / mu_n + mu_s**2 * var_n / mu_n**2) / (k * mu_n**2)++ @property+ def unfloored_variance(self) -> float:+ """Clustered per-event variance: iid variance times the design effect of the CDF"""+ cdf_iid_variance = self.nu * (1 - self.nu) / self.n if self.n > 0 else 0.0+ if cdf_iid_variance <= 0:+ return 0.0+ return self.iid_variance * self.cdf_cluster_variance / cdf_iid_variance++ @property+ def variance(self) -> float:+ """Clustered per-event variance, floored at 1e-5 when n >= 100"""+ variance = self.unfloored_variance+ return max(variance, 1e-5) if self.n >= 100 else variance++ @property+ def has_zero_variance(self) -> bool:+ """True when the outer interval leaves [0, 1] or the variance is not positive"""+ z2 = Z_975**2+ if self.n <= z2 * self.nu / (1 - self.nu) or self.n <= z2 * (1 - self.nu) / self.nu:+ return True+ # The floor in variance turns collapsed bounds (Y_L == Y_U) into 1e-5, so read the value before it.+ return self.unfloored_variance <= 0++ @property+ def standard_error(self) -> float:+ """Standard error of the quantile: SE = √(variance/n)"""+ return float(np.sqrt(self.variance / self.n))++# Type alias for any statistic type-AnyStatistic = SampleMeanStatistic | ProportionStatistic | RatioStatistic+AnyStatistic = SampleMeanStatistic | ProportionStatistic | RatioStatistic | QuantileStatisticdiff --git a/products/experiments/stats/shared/utils.py b/products/experiments/stats/shared/utils.pyindex e661024faabb..eaa275cec8b2 100644--- a/products/experiments/stats/shared/utils.py+++ b/products/experiments/stats/shared/utils.py@@ -7,7 +7,14 @@ by both frequentist and Bayesian methods.import numpy as np-from .statistics import AnyStatistic, ProportionStatistic, RatioStatistic, SampleMeanStatistic, StatisticError+from .statistics import (+ AnyStatistic,+ ProportionStatistic,+ QuantileStatistic,+ RatioStatistic,+ SampleMeanStatistic,+ StatisticError,+)def get_mean(statistic: AnyStatistic) -> float:@@ -18,6 +25,8 @@ def get_mean(statistic: AnyStatistic) -> float:return statistic.proportionelif isinstance(statistic, RatioStatistic):return statistic.ratio+ elif isinstance(statistic, QuantileStatistic):+ return statistic.meanelse:raise StatisticError(f"Unknown statistic type: {type(statistic)}")@@ -64,6 +73,8 @@ def validate_statistic_inputs(statistic: AnyStatistic) -> list[str]:errors.extend(_validate_sample_mean(statistic))elif isinstance(statistic, ProportionStatistic):errors.extend(_validate_proportion(statistic))+ elif isinstance(statistic, QuantileStatistic):+ errors.extend(_validate_quantile(statistic))except Exception as e:errors.append(f"Failed to validate statistic: {str(e)}")@@ -110,6 +121,16 @@ def _validate_proportion(stat: ProportionStatistic) -> list[str]:return errors+def _validate_quantile(stat: QuantileStatistic) -> list[str]:+ """Validate QuantileStatistic."""+ errors = []++ if stat.has_zero_variance:+ errors.append("Quantile has zero variance - not enough events or too many tied values")++ return errors++def validate_test_inputs(treatment_stat: AnyStatistic, control_stat: AnyStatistic) -> None:"""Comprehensive validation of test inputs.diff --git a/products/experiments/stats/tests/test_bayesian.py b/products/experiments/stats/tests/test_bayesian.pyindex ad15bd6b8e2a..e6cbd7a07103 100644--- a/products/experiments/stats/tests/test_bayesian.py+++ b/products/experiments/stats/tests/test_bayesian.py@@ -11,6 +11,7 @@ import pytestfrom unittest import TestCaseimport numpy as np+from parameterized import parameterizedfrom products.experiments.stats.bayesian.method import BayesianConfig, BayesianMethodfrom products.experiments.stats.bayesian.priors import GaussianPrior@@ -28,6 +29,11 @@ from products.experiments.stats.shared.statistics import (SampleMeanStatistic,StatisticError,)+from products.experiments.stats.tests.test_statistics import (+ DEGENERATE_QUANTILE_CASES,+ QUANTILE_CONTROL,+ QUANTILE_TREATMENT,+)class TestGaussianPrior:@@ -313,6 +319,40 @@ class TestBayesianMethod(TestCase):self.assertAlmostEqual(credible_interval_result[0], expected_dict["credible_interval"][0], places=4)self.assertAlmostEqual(credible_interval_result[1], expected_dict["credible_interval"][1], places=4)+ @parameterized.expand(+ [+ (+ "absolute",+ DifferenceType.ABSOLUTE,+ 1.415000000000001,+ (0.08583772923349264, 2.744162270766509),+ 0.9815350910332983,+ ),+ (+ "relative",+ DifferenceType.RELATIVE,+ 0.12982842462611258,+ (0.0031120207966113256, 0.2565448284556138),+ 0.9776835938774414,+ ),+ ]+ )+ def test_gaussian_test_with_quantile_statistic(self, _name, difference_type, effect, interval, chance_to_win):+ method = BayesianMethod(BayesianConfig(ci_level=0.95, difference_type=difference_type))+ result = method.run_test(QUANTILE_TREATMENT, QUANTILE_CONTROL)++ self.assertAlmostEqual(result.effect_size, effect, places=12)+ self.assertAlmostEqual(result.credible_interval[0], interval[0], places=10)+ self.assertAlmostEqual(result.credible_interval[1], interval[1], places=10)+ self.assertAlmostEqual(result.chance_to_win, chance_to_win, places=10)++ @parameterized.expand(DEGENERATE_QUANTILE_CASES)+ def test_rejects_quantile_with_zero_variance(self, _name, treatment, control):+ with self.assertRaisesRegex(+ StatisticError, "Treatment statistic validation failed: Quantile has zero variance"+ ):+ BayesianMethod().run_test(treatment, control)+class TestConvenienceFunctions:"""Tests for convenience functions."""diff --git a/products/experiments/stats/tests/test_frequentist.py b/products/experiments/stats/tests/test_frequentist.pyindex 37a4aa5f7587..dbcf87e68dda 100644--- a/products/experiments/stats/tests/test_frequentist.py+++ b/products/experiments/stats/tests/test_frequentist.py@@ -20,6 +20,11 @@ from products.experiments.stats.shared.statistics import (SampleMeanStatistic,StatisticError,)+from products.experiments.stats.tests.test_statistics import (+ DEGENERATE_QUANTILE_CASES,+ QUANTILE_CONTROL,+ QUANTILE_TREATMENT,+)def create_test_result_dict(result: Any) -> dict[str, Any]:@@ -165,6 +170,47 @@ class TestTwoSidedTTest(TestCase):self.assertAlmostEqual(uplift["mean"], expected_uplift["mean"], places=4)self.assertAlmostEqual(uplift["stddev"], expected_uplift["stddev"], places=4)+ @parameterized.expand(+ [+ (+ "absolute",+ DifferenceType.ABSOLUTE,+ 1.415000000000001,+ (0.08506091978298969, 2.744939080217012),+ 0.0370517987954897,+ ),+ (+ "relative",+ DifferenceType.RELATIVE,+ 0.12982842462611258,+ (0.003037963247587039, 0.2566188860046381),+ 0.044762375592802384,+ ),+ ]+ )+ def test_two_sided_ttest_with_quantile_statistic(self, _name, difference_type, expected, ci, p_value):+ method = FrequentistMethod(FrequentistConfig(alpha=0.05, difference_type=difference_type))+ result = method.run_test(QUANTILE_TREATMENT, QUANTILE_CONTROL)++ self.assertAlmostEqual(result.point_estimate, expected, places=12)+ self.assertAlmostEqual(result.confidence_interval[0], ci[0], places=10)+ self.assertAlmostEqual(result.confidence_interval[1], ci[1], places=10)+ self.assertAlmostEqual(result.p_value, p_value, places=9)++ @parameterized.expand(+ [+ (f"{name}_{mode}", treatment, control, sequential)+ for name, treatment, control in DEGENERATE_QUANTILE_CASES+ for mode, sequential in [("fixed_horizon", False), ("sequential", True)]+ ]+ )+ def test_rejects_quantile_with_zero_variance(self, _name, treatment, control, sequential):+ method = FrequentistMethod(FrequentistConfig(sequential_testing_enabled=sequential))+ with self.assertRaisesRegex(+ StatisticError, "Treatment statistic validation failed: Quantile has zero variance"+ ):+ method.run_test(treatment, control)+class TestSequentialMath(TestCase):"""Unit tests for the sequential testing primitives (rho, half-width, p-value)."""diff --git a/products/experiments/stats/tests/test_statistics.py b/products/experiments/stats/tests/test_statistics.pyindex fc1e135457dc..25092f2bd097 100644--- a/products/experiments/stats/tests/test_statistics.py+++ b/products/experiments/stats/tests/test_statistics.py@@ -1,8 +1,93 @@+from typing import Any+import pytest+from unittest import TestCaseimport numpy as np--from ..shared.statistics import InvalidStatisticError, ProportionStatistic, RatioStatistic, SampleMeanStatistic+from parameterized import parameterized++from ..shared.statistics import (+ QUANTILE_N_STAR_GRID,+ InvalidStatisticError,+ ProportionStatistic,+ QuantileBounds,+ QuantileGridLevel,+ QuantileStatistic,+ RatioStatistic,+ SampleMeanStatistic,+ choose_quantile_bounds,+ quantile_grid_levels,+)++# Sufficient statistics of a seeded synthetic dataset: 400 users per arm, Poisson(3) events+# per user (some users have none), lognormal values with a per-user random effect, nu = 0.9.+QUANTILE_CONTROL = QuantileStatistic(+ nu=0.9,+ n=1263,+ n_clusters=400,+ quantile=10.899,+ quantile_lower=10.34,+ quantile_upper=11.633,+ n_star=800,+ s_sum=1137,+ s_sum_squares=4443,+ n_sum=1263,+ n_sum_squares=5193,+ s_n_sum_of_products=4684,+)+QUANTILE_TREATMENT = QuantileStatistic(+ nu=0.9,+ n=1217,+ n_clusters=400,+ quantile=12.314,+ quantile_lower=11.178,+ quantile_upper=13.101,+ n_star=800,+ s_sum=1096,+ s_sum_squares=4138,+ n_sum=1217,+ n_sum_squares=4793,+ s_n_sum_of_products=4330,+)+++def _iid_quantile(nu: float, n: int, lower: float, upper: float, n_clusters: int | None = None) -> QuantileStatistic:+ # One event per user, so the design effect is K / (K - 1).+ k = n if n_clusters is None else n_clusters+ return QuantileStatistic(+ nu=nu,+ n=n,+ n_clusters=k,+ quantile=(lower + upper) / 2,+ quantile_lower=lower,+ quantile_upper=upper,+ n_star=max([n_star for n_star in QUANTILE_N_STAR_GRID if n_star <= n], default=QUANTILE_N_STAR_GRID[0]),+ s_sum=nu * n,+ s_sum_squares=nu * n,+ n_sum=n,+ n_sum_squares=n,+ s_n_sum_of_products=nu * n,+ )+++# (name, treatment, control) pairs in which the treatment quantile has no usable variance.+DEGENERATE_QUANTILE_CASES = [+ (+ "collapsed_bounds",+ _iid_quantile(nu=0.6, n=700, lower=4.0, upper=4.0),+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),+ ),+ (+ "tail_too_small_for_quantile",+ _iid_quantile(nu=0.98, n=150, lower=3.9, upper=4.1),+ _iid_quantile(nu=0.98, n=6_000, lower=3.9, upper=4.1),+ ),+ (+ "single_cluster",+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1, n_clusters=1),+ _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),+ ),+]class TestSampleMeanStatistic:@@ -176,3 +261,93 @@ class TestStatisticIntegration:assert abs(sample_mean.standard_error - expected_se_mean) < 1e-10assert abs(proportion.standard_error - expected_se_prop) < 1e-10+++class TestQuantileStatistic(TestCase):+ @parameterized.expand(+ [+ ("control", QUANTILE_CONTROL, 86.97348405307501, 0.00013708395792512163, 167.31479431374606),+ ("treatment", QUANTILE_TREATMENT, 192.3691290860306, 0.0001531846058978492, 398.4726987554284),+ ]+ )+ def test_clustered_variance(self, _name, stat, iid_variance, cdf_cluster_variance, variance):+ assert stat.mean == stat.quantile+ assert stat.iid_variance == pytest.approx(iid_variance, rel=1e-12)+ assert stat.cdf_cluster_variance == pytest.approx(cdf_cluster_variance, rel=1e-12)+ assert stat.variance == pytest.approx(variance, rel=1e-12)+ assert stat.standard_error == pytest.approx(np.sqrt(variance / stat.n), rel=1e-12)+ assert not stat.has_zero_variance++ @parameterized.expand(+ [+ ("floored_at_n_100_or_more", 400, 1e-5),+ ("not_floored_below_n_100", 50, 6.507944290644809e-08),+ ]+ )+ def test_variance_floor(self, _name, n, expected_variance):+ stat = _iid_quantile(nu=0.5, n=n, lower=10.0, upper=10.0001)+ assert stat.variance == pytest.approx(expected_variance, rel=1e-9)++ @parameterized.expand(+ [+ ("collapsed_bounds_large_n", 0.75, 50_000_000, 8.0, 8.0, None, True),+ ("collapsed_bounds_small_n", 0.75, 2_000, 8.0, 8.0, None, True),+ ("open_bounds_large_n", 0.75, 50_000_000, 7.98, 8.02, None, False),+ ("interval_leaves_unit_range", 0.99, 300, 7.98, 8.02, None, True),+ ("single_cluster", 0.75, 2_000, 7.98, 8.02, 1, True),+ ]+ )+ def test_has_zero_variance(self, _name, nu, n, lower, upper, n_clusters, expected):+ stat = _iid_quantile(nu=nu, n=n, lower=lower, upper=upper, n_clusters=n_clusters)+ assert stat.has_zero_variance is expected++ @parameterized.expand(+ [+ ("nu_out_of_range", {"nu": 1.0}),+ ("no_events", {"n": 0, "n_sum": 0}),+ ("no_clusters", {"n_clusters": 0}),+ ("n_sum_differs_from_n", {"n_sum": 1262}),+ ("s_sum_above_n_sum", {"s_sum": 1264}),+ ("lower_above_upper", {"quantile_lower": 12.0}),+ ]+ )+ def test_validation_errors(self, _name, overrides: dict[str, Any]):+ fields = {name: getattr(QUANTILE_CONTROL, name) for name in QUANTILE_CONTROL.__dataclass_fields__}+ with pytest.raises(InvalidStatisticError):+ QuantileStatistic(**{**fields, **overrides})+++class TestQuantileGridLevels(TestCase):+ @parameterized.expand(+ [+ ("nu_75_smallest_grid", 0.75, 0, (100, 0.6651310699, 0.8348689301)),+ ("nu_75_n_star_1600", 0.75, 4, (1600, 0.7287827675, 0.7712172325)),+ ("upper_clamped_below_one", 0.99, 0, (100, 0.9704986046, 0.99999999)),+ ]+ )+ def test_levels(self, _name, nu, index, expected):+ levels = quantile_grid_levels(nu)+ assert [level.n_star for level in levels] == QUANTILE_N_STAR_GRID+ n_star, lower_level, upper_level = expected+ assert levels[index] == QuantileGridLevel(n_star=n_star, lower_level=lower_level, upper_level=upper_level)+++class TestChooseQuantileBounds(TestCase):+ BOUNDS = [(100, 1.0, 3.0), (200, 1.5, 2.5), (400, 1.8, 2.2), (800, 2.0, 2.0)]++ @staticmethod+ def _bounds(n_star: int, lower: float, upper: float) -> QuantileBounds:+ return QuantileBounds(n_star=n_star, lower=lower, upper=upper)++ @parameterized.expand(+ [+ ("largest_n_star_below_n", 300, BOUNDS, (200, 1.5, 2.5)),+ ("n_star_equal_to_n", 400, BOUNDS, (400, 1.8, 2.2)),+ ("smallest_grid_size_when_n_below_it", 40, BOUNDS, (100, 1.0, 3.0)),+ ("steps_down_when_bounds_collapse", 1000, BOUNDS, (400, 1.8, 2.2)),+ ("none_when_all_bounds_collapse", 1000, [(100, 2.0, 2.0), (200, 2.0, 2.0)], None),+ ]+ )+ def test_choose(self, _name, n, bounds, expected):+ chosen = choose_quantile_bounds(n, [self._bounds(*b) for b in bounds])+ assert chosen == (self._bounds(*expected) if expected is not None else None)