Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.7 KB
Raw Blame
from typing import Any

import pytest
from unittest import TestCase

import numpy as np
from parameterized import parameterized

from ..shared.statistics import (
    QUANTILE_N_STAR_GRID,
    InvalidStatisticError,
    ProportionStatistic,
    QuantileBounds,
    QuantileGridLevel,
    QuantileStatistic,
    RatioStatistic,
    SampleMeanStatistic,
    choose_quantile_bounds,
    quantile_grid_levels,
)

# Sufficient statistics of a seeded synthetic dataset: 400 users per arm, Poisson(3) events
# per user (some users have none), lognormal values with a per-user random effect, nu = 0.9.
QUANTILE_CONTROL = QuantileStatistic(
    nu=0.9,
    n=1263,
    n_clusters=400,
    quantile=10.899,
    quantile_lower=10.34,
    quantile_upper=11.633,
    n_star=800,
    s_sum=1137,
    s_sum_squares=4443,
    n_sum=1263,
    n_sum_squares=5193,
    s_n_sum_of_products=4684,
)
QUANTILE_TREATMENT = QuantileStatistic(
    nu=0.9,
    n=1217,
    n_clusters=400,
    quantile=12.314,
    quantile_lower=11.178,
    quantile_upper=13.101,
    n_star=800,
    s_sum=1096,
    s_sum_squares=4138,
    n_sum=1217,
    n_sum_squares=4793,
    s_n_sum_of_products=4330,
)


def _iid_quantile(nu: float, n: int, lower: float, upper: float, n_clusters: int | None = None) -> QuantileStatistic:
    # One event per user, so the design effect is K / (K - 1).
    k = n if n_clusters is None else n_clusters
    return QuantileStatistic(
        nu=nu,
        n=n,
        n_clusters=k,
        quantile=(lower + upper) / 2,
        quantile_lower=lower,
        quantile_upper=upper,
        n_star=max([n_star for n_star in QUANTILE_N_STAR_GRID if n_star <= n], default=QUANTILE_N_STAR_GRID[0]),
        s_sum=nu * n,
        s_sum_squares=nu * n,
        n_sum=n,
        n_sum_squares=n,
        s_n_sum_of_products=nu * n,
    )


# (name, treatment, control) pairs in which the treatment quantile has no usable variance.
DEGENERATE_QUANTILE_CASES = [
    (
        "collapsed_bounds",
        _iid_quantile(nu=0.6, n=700, lower=4.0, upper=4.0),
        _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),
    ),
    (
        "tail_too_small_for_quantile",
        _iid_quantile(nu=0.98, n=150, lower=3.9, upper=4.1),
        _iid_quantile(nu=0.98, n=6_000, lower=3.9, upper=4.1),
    ),
    (
        "single_cluster",
        _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1, n_clusters=1),
        _iid_quantile(nu=0.6, n=700, lower=3.9, upper=4.1),
    ),
]


class TestSampleMeanStatistic:
    """Tests for SampleMeanStatistic."""

    def test_basic_calculations(self):
        """Test basic mean and variance calculations."""
        # Test case: [1, 2, 3, 4, 5]
        stat = SampleMeanStatistic(n=5, sum=15, sum_squares=55)

        assert stat.mean == 3.0
        assert stat.variance == 2.5  # Sample variance
        assert abs(stat.standard_error - np.sqrt(2.5 / 5)) < 1e-10

    def test_single_observation(self):
        """Test with single observation."""
        stat = SampleMeanStatistic(n=1, sum=10, sum_squares=100)

        assert stat.mean == 10.0
        assert stat.variance == 0.0
        assert stat.standard_error == 0.0

    def test_zero_variance(self):
        """Test with all identical values."""
        # All values are 5: [5, 5, 5]
        stat = SampleMeanStatistic(n=3, sum=15, sum_squares=75)

        assert stat.mean == 5.0
        assert stat.variance == 0.0
        assert stat.standard_error == 0.0

    def test_validation_errors(self):
        """Test input validation."""
        # Negative sample size
        with pytest.raises(InvalidStatisticError):
            SampleMeanStatistic(n=-1, sum=10, sum_squares=100)

        # Zero sample size
        with pytest.raises(InvalidStatisticError):
            SampleMeanStatistic(n=0, sum=10, sum_squares=100)

        # Invalid sum_squares (too small)
        with pytest.raises(InvalidStatisticError):
            SampleMeanStatistic(n=5, sum=15, sum_squares=40)  # 40 < 15²/5 = 45


class TestProportionStatistic:
    """Tests for ProportionStatistic."""

    def test_basic_calculations(self):
        """Test basic proportion calculations."""
        stat = ProportionStatistic(n=100, sum=25)

        assert stat.proportion == 0.25
        assert stat.variance == 0.25 * 0.75  # p(1-p)
        assert abs(stat.standard_error - np.sqrt(0.25 * 0.75 / 100)) < 1e-10

    def test_extreme_proportions(self):
        """Test with extreme proportions."""
        # All successes
        stat = ProportionStatistic(n=10, sum=10)
        assert stat.proportion == 1.0
        assert stat.variance == 0.0

        # No successes
        stat = ProportionStatistic(n=10, sum=0)
        assert stat.proportion == 0.0
        assert stat.variance == 0.0

    def test_validation_errors(self):
        """Test input validation."""
        # Negative sample size
        with pytest.raises(InvalidStatisticError):
            ProportionStatistic(n=-1, sum=5)

        # Negative successes
        with pytest.raises(InvalidStatisticError):
            ProportionStatistic(n=10, sum=-1)

        # More successes than trials
        with pytest.raises(InvalidStatisticError):
            ProportionStatistic(n=10, sum=15)


class TestRatioStatistic:
    """Tests for RatioStatistic."""

    def test_basic_calculations(self):
        """Test basic ratio calculations."""
        # Revenue per user: revenue=[10, 20, 30], users=[1, 1, 1]
        numerator = SampleMeanStatistic(n=3, sum=60, sum_squares=1400)  # Revenue
        denominator = SampleMeanStatistic(n=3, sum=3, sum_squares=3)  # Users

        stat = RatioStatistic(
            n=3,
            m_statistic=numerator,
            d_statistic=denominator,
            m_d_sum_of_products=60,  # sum(revenue * users)
        )

        assert stat.ratio == 20.0  # 60 / 3

        # Test covariance calculation
        expected_cov = (60 - 60 * 3 / 3) / (3 - 1)  # Should be 0
        assert abs(stat.covariance - expected_cov) < 1e-10

    def test_with_proportion_denominator(self):
        """Test ratio with proportion denominator (e.g., revenue per conversion)."""
        revenue = SampleMeanStatistic(n=100, sum=500, sum_squares=5000)
        conversions = ProportionStatistic(n=100, sum=20)  # 20% conversion rate

        stat = RatioStatistic(
            n=100,
            m_statistic=revenue,
            d_statistic=conversions,
            m_d_sum_of_products=100,  # sum(revenue * conversion_indicator)
        )

        assert stat.ratio == 500 / 20  # Total revenue / total conversions

    def test_validation_errors(self):
        """Test input validation."""
        numerator = SampleMeanStatistic(n=5, sum=25, sum_squares=125)
        denominator = SampleMeanStatistic(n=5, sum=0, sum_squares=0)  # Zero denominator

        # Zero denominator
        with pytest.raises(InvalidStatisticError):
            RatioStatistic(n=5, m_statistic=numerator, d_statistic=denominator, m_d_sum_of_products=0)

        # Mismatched sample sizes
        denominator_wrong_n = SampleMeanStatistic(n=3, sum=15, sum_squares=75)
        with pytest.raises(InvalidStatisticError):
            RatioStatistic(n=5, m_statistic=numerator, d_statistic=denominator_wrong_n, m_d_sum_of_products=0)


class TestStatisticIntegration:
    """Integration tests across statistic types."""

    def test_mean_extraction(self):
        """Test that all statistics can provide a mean value."""
        sample_mean = SampleMeanStatistic(n=10, sum=50, sum_squares=300)
        proportion = ProportionStatistic(n=100, sum=25)

        assert hasattr(sample_mean, "mean")
        assert hasattr(proportion, "proportion")

        # Both should be usable in calculations
        assert sample_mean.mean == 5.0
        assert proportion.proportion == 0.25

    def test_variance_extraction(self):
        """Test that all statistics can provide variance."""
        sample_mean = SampleMeanStatistic(n=10, sum=50, sum_squares=300)
        proportion = ProportionStatistic(n=100, sum=25)

        assert hasattr(sample_mean, "variance")
        assert hasattr(proportion, "variance")

        # Both should be non-negative
        assert sample_mean.variance >= 0
        assert proportion.variance >= 0

    def test_standard_error_calculation(self):
        """Test standard error calculations across types."""
        sample_mean = SampleMeanStatistic(n=10, sum=50, sum_squares=300)
        proportion = ProportionStatistic(n=100, sum=25)

        # Standard error should be sqrt(variance / n)
        expected_se_mean = np.sqrt(sample_mean.variance / sample_mean.n)
        expected_se_prop = np.sqrt(proportion.variance / proportion.n)

        assert abs(sample_mean.standard_error - expected_se_mean) < 1e-10
        assert abs(proportion.standard_error - expected_se_prop) < 1e-10


class TestQuantileStatistic(TestCase):
    @parameterized.expand(
        [
            ("control", QUANTILE_CONTROL, 86.97348405307501, 0.00013708395792512163, 167.31479431374606),
            ("treatment", QUANTILE_TREATMENT, 192.3691290860306, 0.0001531846058978492, 398.4726987554284),
        ]
    )
    def test_clustered_variance(self, _name, stat, iid_variance, cdf_cluster_variance, variance):
        assert stat.mean == stat.quantile
        assert stat.iid_variance == pytest.approx(iid_variance, rel=1e-12)
        assert stat.cdf_cluster_variance == pytest.approx(cdf_cluster_variance, rel=1e-12)
        assert stat.variance == pytest.approx(variance, rel=1e-12)
        assert stat.standard_error == pytest.approx(np.sqrt(variance / stat.n), rel=1e-12)
        assert not stat.has_zero_variance

    @parameterized.expand(
        [
            ("floored_at_n_100_or_more", 400, 1e-5),
            ("not_floored_below_n_100", 50, 6.507944290644809e-08),
        ]
    )
    def test_variance_floor(self, _name, n, expected_variance):
        stat = _iid_quantile(nu=0.5, n=n, lower=10.0, upper=10.0001)
        assert stat.variance == pytest.approx(expected_variance, rel=1e-9)

    @parameterized.expand(
        [
            ("collapsed_bounds_large_n", 0.75, 50_000_000, 8.0, 8.0, None, True),
            ("collapsed_bounds_small_n", 0.75, 2_000, 8.0, 8.0, None, True),
            ("open_bounds_large_n", 0.75, 50_000_000, 7.98, 8.02, None, False),
            ("interval_leaves_unit_range", 0.99, 300, 7.98, 8.02, None, True),
            ("single_cluster", 0.75, 2_000, 7.98, 8.02, 1, True),
        ]
    )
    def test_has_zero_variance(self, _name, nu, n, lower, upper, n_clusters, expected):
        stat = _iid_quantile(nu=nu, n=n, lower=lower, upper=upper, n_clusters=n_clusters)
        assert stat.has_zero_variance is expected

    @parameterized.expand(
        [
            ("nu_out_of_range", {"nu": 1.0}),
            ("no_events", {"n": 0, "n_sum": 0}),
            ("no_clusters", {"n_clusters": 0}),
            ("n_sum_differs_from_n", {"n_sum": 1262}),
            ("s_sum_above_n_sum", {"s_sum": 1264}),
            ("lower_above_upper", {"quantile_lower": 12.0}),
        ]
    )
    def test_validation_errors(self, _name, overrides: dict[str, Any]):
        fields = {name: getattr(QUANTILE_CONTROL, name) for name in QUANTILE_CONTROL.__dataclass_fields__}
        with pytest.raises(InvalidStatisticError):
            QuantileStatistic(**{**fields, **overrides})


class TestQuantileGridLevels(TestCase):
    @parameterized.expand(
        [
            ("nu_75_smallest_grid", 0.75, 0, (100, 0.6651310699, 0.8348689301)),
            ("nu_75_n_star_1600", 0.75, 4, (1600, 0.7287827675, 0.7712172325)),
            ("upper_clamped_below_one", 0.99, 0, (100, 0.9704986046, 0.99999999)),
        ]
    )
    def test_levels(self, _name, nu, index, expected):
        levels = quantile_grid_levels(nu)
        assert [level.n_star for level in levels] == QUANTILE_N_STAR_GRID
        n_star, lower_level, upper_level = expected
        assert levels[index] == QuantileGridLevel(n_star=n_star, lower_level=lower_level, upper_level=upper_level)


class TestChooseQuantileBounds(TestCase):
    BOUNDS = [(100, 1.0, 3.0), (200, 1.5, 2.5), (400, 1.8, 2.2), (800, 2.0, 2.0)]

    @staticmethod
    def _bounds(n_star: int, lower: float, upper: float) -> QuantileBounds:
        return QuantileBounds(n_star=n_star, lower=lower, upper=upper)

    @parameterized.expand(
        [
            ("largest_n_star_below_n", 300, BOUNDS, (200, 1.5, 2.5)),
            ("n_star_equal_to_n", 400, BOUNDS, (400, 1.8, 2.2)),
            ("smallest_grid_size_when_n_below_it", 40, BOUNDS, (100, 1.0, 3.0)),
            ("steps_down_when_bounds_collapse", 1000, BOUNDS, (400, 1.8, 2.2)),
            ("none_when_all_bounds_collapse", 1000, [(100, 2.0, 2.0), (200, 2.0, 2.0)], None),
        ]
    )
    def test_choose(self, _name, n, bounds, expected):
        chosen = choose_quantile_bounds(n, [self._bounds(*b) for b in bounds])
        assert chosen == (self._bounds(*expected) if expected is not None else None)