:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import math
from typing import Any, cast
from unittest import TestCase
from parameterized import parameterized
from products.experiments.stats.frequentist.method import FrequentistConfig, FrequentistMethod
from products.experiments.stats.frequentist.tests import SequentialTwoSidedTTest
from products.experiments.stats.frequentist.utils import (
calculate_variance_pooled,
sequential_interval_halfwidth,
sequential_p_value,
sequential_rho,
)
from products.experiments.stats.shared.enums import DifferenceType
from products.experiments.stats.shared.statistics import (
ProportionStatistic,
RatioStatistic,
SampleMeanStatistic,
StatisticError,
)
from products.experiments.stats.tests.test_statistics import (
DEGENERATE_QUANTILE_CASES,
QUANTILE_CONTROL,
QUANTILE_TREATMENT,
)
def create_test_result_dict(result: Any) -> dict[str, Any]:
"""Convert TestResult to dictionary for easy interpretation."""
return {
"expected": result.point_estimate,
"ci": [result.confidence_interval[0], result.confidence_interval[1]],
"p_value": result.p_value,
"error_message": None,
"uplift": {
"dist": "normal",
"mean": result.point_estimate,
"stddev": (result.confidence_interval[1] - result.confidence_interval[0]) / (2 * 1.96) ** 2
if result.confidence_interval[1] != float("inf") and result.confidence_interval[0] != float("-inf")
else None,
},
}
def get_ci(result_dict: dict[str, Any]) -> list[Any]:
return cast(list[Any], result_dict["ci"])
def get_expected_uplift(expected_dict: dict[str, Any]) -> dict[str, Any]:
return cast(dict[str, Any], expected_dict["uplift"])
class TestTwoSidedTTest(TestCase):
@staticmethod
def _uplift(result_dict: dict[str, Any]) -> dict[str, Any]:
return cast(dict[str, Any], result_dict["uplift"])
def test_two_sided_ttest_with_sample_mean(self):
"""Test basic two-sided t-test with sample mean statistics."""
# imported here so that pytest doesn't try to discover tests inside it
from products.experiments.stats.frequentist.method import TestType
stat_a = SampleMeanStatistic(sum=1922.7, sum_squares=94698.29, n=2461)
stat_b = SampleMeanStatistic(sum=1196.87, sum_squares=37377.9767, n=2507)
config = FrequentistConfig(alpha=0.05, test_type=TestType.TWO_SIDED, difference_type=DifferenceType.RELATIVE)
method = FrequentistMethod(config)
result = method.run_test(stat_a, stat_b)
result_dict = create_test_result_dict(result)
expected_dict: dict[str, Any] = {
"expected": 0.63646,
"ci": [-0.0875, 1.36048],
"p_value": 0.08487,
"error_message": None,
"uplift": {"dist": "normal", "mean": 0.636467, "stddev": 0.094233},
}
# Compare the key values
self.assertAlmostEqual(result_dict["expected"], expected_dict["expected"], places=4)
self.assertAlmostEqual(result_dict["p_value"], expected_dict["p_value"], places=4)
ci = get_ci(result_dict)
self.assertAlmostEqual(ci[0], expected_dict["ci"][0], places=4)
self.assertAlmostEqual(ci[1], expected_dict["ci"][1], places=4)
uplift = self._uplift(result_dict)
expected_uplift = get_expected_uplift(expected_dict)
self.assertAlmostEqual(uplift["mean"], expected_uplift["mean"], places=4)
self.assertAlmostEqual(uplift["stddev"], expected_uplift["stddev"], places=4)
def test_two_sided_ttest_with_sample_proportion(self):
"""Test basic two-sided t-test with sample proportion statistics."""
# imported here so that pytest doesn't try to discover tests inside it
from products.experiments.stats.frequentist.method import TestType
stat_a = ProportionStatistic(sum=62, n=1471)
stat_b = ProportionStatistic(sum=87, n=1529)
config = FrequentistConfig(alpha=0.05, test_type=TestType.TWO_SIDED, difference_type=DifferenceType.RELATIVE)
method = FrequentistMethod(config)
result = method.run_test(stat_a, stat_b)
result_dict = create_test_result_dict(result)
expected_dict: dict[str, Any] = {
"expected": -0.25925,
"ci": [-0.49475, -0.02376],
"p_value": 0.030960,
"error_message": None,
"uplift": {"dist": "normal", "mean": -0.25925, "stddev": 0.030650},
}
# Compare the key values
self.assertAlmostEqual(result_dict["expected"], expected_dict["expected"], places=4)
self.assertAlmostEqual(result_dict["p_value"], expected_dict["p_value"], places=4)
ci = get_ci(result_dict)
self.assertAlmostEqual(ci[0], expected_dict["ci"][0], places=4)
self.assertAlmostEqual(ci[1], expected_dict["ci"][1], places=4)
uplift = self._uplift(result_dict)
expected_uplift = get_expected_uplift(expected_dict)
self.assertAlmostEqual(uplift["mean"], expected_uplift["mean"], places=4)
self.assertAlmostEqual(uplift["stddev"], expected_uplift["stddev"], places=4)
def test_two_sided_ttest_with_ratio_statistic(self):
"""Test basic two-sided t-test with ratio statistics."""
# imported here so that pytest doesn't try to discover tests inside it
from products.experiments.stats.frequentist.method import TestType
# treatment
stat_a_n = 2034
stat_a = RatioStatistic(
n=stat_a_n,
m_statistic=SampleMeanStatistic(n=stat_a_n, sum=99673.9364269569, sum_squares=11298745.182728939),
d_statistic=SampleMeanStatistic(n=stat_a_n, sum=947, sum_squares=947),
m_d_sum_of_products=99673.9364269569,
)
# control
stat_b_n = 1966
stat_b = RatioStatistic(
n=stat_b_n,
m_statistic=SampleMeanStatistic(n=stat_b_n, sum=94605.79858780127, sum_squares=10463129.505392816),
d_statistic=SampleMeanStatistic(n=stat_b_n, sum=936, sum_squares=936),
m_d_sum_of_products=94605.79858780127,
)
config = FrequentistConfig(alpha=0.05, test_type=TestType.TWO_SIDED, difference_type=DifferenceType.RELATIVE)
method = FrequentistMethod(config)
result = method.run_test(stat_a, stat_b)
result_dict = create_test_result_dict(result)
expected_dict: dict[str, Any] = {
"expected": 0.041333,
"ci": [0.01378609, 0.0689],
"p_value": 0.0032826,
"error_message": None,
"uplift": {"dist": "normal", "mean": 0.0413, "stddev": 0.00358537},
}
# Compare the key values
self.assertAlmostEqual(result_dict["expected"], expected_dict["expected"], places=4)
self.assertAlmostEqual(result_dict["p_value"], expected_dict["p_value"], places=4)
ci = get_ci(result_dict)
self.assertAlmostEqual(ci[0], expected_dict["ci"][0], places=4)
self.assertAlmostEqual(ci[1], expected_dict["ci"][1], places=4)
uplift = self._uplift(result_dict)
expected_uplift = get_expected_uplift(expected_dict)
self.assertAlmostEqual(uplift["mean"], expected_uplift["mean"], places=4)
self.assertAlmostEqual(uplift["stddev"], expected_uplift["stddev"], places=4)
@parameterized.expand(
[
(
"absolute",
DifferenceType.ABSOLUTE,
1.415000000000001,
(0.08506091978298969, 2.744939080217012),
0.0370517987954897,
),
(
"relative",
DifferenceType.RELATIVE,
0.12982842462611258,
(0.003037963247587039, 0.2566188860046381),
0.044762375592802384,
),
]
)
def test_two_sided_ttest_with_quantile_statistic(self, _name, difference_type, expected, ci, p_value):
method = FrequentistMethod(FrequentistConfig(alpha=0.05, difference_type=difference_type))
result = method.run_test(QUANTILE_TREATMENT, QUANTILE_CONTROL)
self.assertAlmostEqual(result.point_estimate, expected, places=12)
self.assertAlmostEqual(result.confidence_interval[0], ci[0], places=10)
self.assertAlmostEqual(result.confidence_interval[1], ci[1], places=10)
self.assertAlmostEqual(result.p_value, p_value, places=9)
@parameterized.expand(
[
(f"{name}_{mode}", treatment, control, sequential)
for name, treatment, control in DEGENERATE_QUANTILE_CASES
for mode, sequential in [("fixed_horizon", False), ("sequential", True)]
]
)
def test_rejects_quantile_with_zero_variance(self, _name, treatment, control, sequential):
method = FrequentistMethod(FrequentistConfig(sequential_testing_enabled=sequential))
with self.assertRaisesRegex(
StatisticError, "Treatment statistic validation failed: Quantile has zero variance"
):
method.run_test(treatment, control)
class TestSequentialMath(TestCase):
"""Unit tests for the sequential testing primitives (rho, half-width, p-value)."""
def test_sequential_rho_matches_formula(self):
alpha = 0.05
n_tuning = 5000.0
rho = sequential_rho(alpha, n_tuning)
log_alpha = math.log(alpha)
expected = math.sqrt((-2 * log_alpha + math.log(-2 * log_alpha + 1)) / n_tuning)
self.assertAlmostEqual(rho, expected, places=10)
def test_sequential_rho_rejects_invalid_alpha(self):
with self.assertRaises(StatisticError):
sequential_rho(0.0, 5000)
with self.assertRaises(StatisticError):
sequential_rho(1.0, 5000)
def test_sequential_rho_rejects_nonpositive_tuning_parameter(self):
with self.assertRaises(StatisticError):
sequential_rho(0.05, 0)
with self.assertRaises(StatisticError):
sequential_rho(0.05, -1)
def test_sequential_halfwidth_wider_than_fixed_horizon(self):
# Per-observation variance sigma^2 = 1, total n = 5000.
# Fixed-horizon halfwidth (1.96 * SE) uses SE = sqrt(sigma^2/n).
s2 = 1.0
n = 5000
alpha = 0.05
seq_halfwidth = sequential_interval_halfwidth(s2, n, 5000, alpha)
fixed_halfwidth = 1.96 * math.sqrt(s2 / n)
self.assertGreater(seq_halfwidth, fixed_halfwidth)
def test_sequential_p_value_clamps_to_one_with_small_n(self):
# Tiny effect, tiny n: e-value < 1 so p-value clamps to 1.
p = sequential_p_value(
point_estimate=0.01,
pooled_variance=1.0,
n=10,
sequential_tuning_parameter=5000,
alpha=0.05,
)
self.assertEqual(p, 1.0)
def test_sequential_p_value_drops_with_strong_evidence(self):
# Strong, well-powered effect should produce a small always-valid p-value.
# SE^2 = 0.0001 (so SE = 0.01) means a 0.5 effect is ~50 standard errors out.
p_strong = sequential_p_value(
point_estimate=0.5,
pooled_variance=0.0001,
n=10000,
sequential_tuning_parameter=5000,
alpha=0.05,
)
self.assertLess(p_strong, 0.05)
class TestSequentialTwoSidedTTest(TestCase):
"""End-to-end behavior of SequentialTwoSidedTTest via FrequentistMethod."""
def test_sequential_ci_is_wider_than_fixed_horizon_ci(self):
from products.experiments.stats.frequentist.method import TestType
stat_a = SampleMeanStatistic(sum=1922.7, sum_squares=94698.29, n=2461)
stat_b = SampleMeanStatistic(sum=1196.87, sum_squares=37377.9767, n=2507)
config_fixed = FrequentistConfig(
alpha=0.05, test_type=TestType.TWO_SIDED, difference_type=DifferenceType.RELATIVE
)
config_seq = FrequentistConfig(
alpha=0.05,
test_type=TestType.TWO_SIDED,
difference_type=DifferenceType.RELATIVE,
sequential_testing_enabled=True,
sequential_tuning_parameter=5000,
)
fixed_result = FrequentistMethod(config_fixed).run_test(stat_a, stat_b)
seq_result = FrequentistMethod(config_seq).run_test(stat_a, stat_b)
fixed_width = fixed_result.confidence_interval[1] - fixed_result.confidence_interval[0]
seq_width = seq_result.confidence_interval[1] - seq_result.confidence_interval[0]
self.assertGreater(seq_width, fixed_width)
self.assertAlmostEqual(seq_result.point_estimate, fixed_result.point_estimate, places=6)
self.assertEqual(seq_result.test_type, "sequential_two_sided")
self.assertTrue(math.isnan(seq_result.degrees_of_freedom))
self.assertTrue(math.isnan(seq_result.test_statistic))
def test_sequential_p_value_with_zero_effect_returns_one(self):
from products.experiments.stats.frequentist.method import TestType
stat_a = ProportionStatistic(sum=100, n=2000)
stat_b = ProportionStatistic(sum=100, n=2000)
config = FrequentistConfig(
alpha=0.05,
test_type=TestType.TWO_SIDED,
difference_type=DifferenceType.ABSOLUTE,
sequential_testing_enabled=True,
sequential_tuning_parameter=5000,
)
result = FrequentistMethod(config).run_test(stat_a, stat_b)
self.assertEqual(result.p_value, 1.0)
self.assertFalse(result.is_significant)
def test_invalid_tuning_parameter_raises(self):
with self.assertRaises(StatisticError):
SequentialTwoSidedTTest(alpha=0.05, sequential_tuning_parameter=0)
class TestSequentialTwoSidedTTestGoldenValues(TestCase):
"""Golden fixtures for the two-sided sequential test.
The implementation was validated to reproduce the published reference values of
Waudby-Smith et al. 2023; these fixtures then lock in that behavior on our own
chosen inputs, guarding the end-to-end computation (point estimate, confidence
sequence, p-value, and the standard error the CI is built from) against drift.
run_test takes (treatment, control), so the higher-mean stat is passed first.
"""
@parameterized.expand(
[
(
"sample_mean_tuning_1000",
SampleMeanStatistic(sum=2500.0, sum_squares=135000.0, n=3500),
SampleMeanStatistic(sum=1400.0, sum_squares=52000.0, n=3000),
1000,
0.53061,
(-0.53230, 1.59353),
0.33368,
1.0,
),
(
"proportion_default_tuning",
ProportionStatistic(sum=2600, n=3500),
ProportionStatistic(sum=1450, n=3050),
5000,
0.56256,
(0.46064, 0.66448),
0.03354,
# Very strong evidence: the e-value is astronomical, so the p-value is a
# near-zero (~6e-56) that rounds to 0.0 at 5 decimals. The log-space form
# keeps the accurate tiny value instead of underflowing 1/e-value to 0.
0.0,
),
]
)
def test_matches_reference_values(self, _name, treatment, control, tuning_parameter, expected, ci, stddev, p_value):
result = SequentialTwoSidedTTest(alpha=0.05, sequential_tuning_parameter=tuning_parameter).run_test(
treatment, control, DifferenceType.RELATIVE
)
self.assertAlmostEqual(result.point_estimate, expected, places=5)
self.assertAlmostEqual(result.confidence_interval[0], ci[0], places=5)
self.assertAlmostEqual(result.confidence_interval[1], ci[1], places=5)
self.assertAlmostEqual(result.p_value, p_value, places=5)
# Assert the standard error directly (the SE the sequential CI is built from) so a
# variance regression is pinpointed, not just inferred from a shifted CI.
standard_error = math.sqrt(calculate_variance_pooled(treatment, control, DifferenceType.RELATIVE))
self.assertAlmostEqual(standard_error, stddev, places=5)
def test_interval_narrowest_near_true_sample_size(self):
# Same stats as the sample-mean fixture; true total n = 3500 + 3000 = 6500. The
# confidence sequence is tightest when the tuning parameter ~ true n and widens as
# it badly under- (10) or over-estimates (10000) it.
treatment = SampleMeanStatistic(sum=2500.0, sum_squares=135000.0, n=3500)
control = SampleMeanStatistic(sum=1400.0, sum_squares=52000.0, n=3000)
def interval(tuning_parameter: float) -> tuple[float, float]:
test = SequentialTwoSidedTTest(alpha=0.05, sequential_tuning_parameter=tuning_parameter)
return test.run_test(treatment, control, DifferenceType.RELATIVE).confidence_interval
below = interval(10)
near = interval(6500)
above = interval(10000)
# Under-estimating the sample size gives the widest interval.
self.assertLess(below[0], near[0])
self.assertGreater(below[1], near[1])
self.assertLess(below[0], above[0])
self.assertGreater(below[1], above[1])
# Over-estimating is wider than estimating well, but narrower than under-estimating.
self.assertLess(above[0], near[0])
self.assertGreater(above[1], near[1])