Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.7 KB
Raw Blame
"""
Shared statistical utilities for A/B testing.

This module provides fundamental statistical utilities that are used
by both frequentist and Bayesian methods.
"""

import numpy as np

from .statistics import (
    AnyStatistic,
    ProportionStatistic,
    QuantileStatistic,
    RatioStatistic,
    SampleMeanStatistic,
    StatisticError,
)


def get_mean(statistic: AnyStatistic) -> float:
    """Extract the mean/central value from any statistic type."""
    if isinstance(statistic, SampleMeanStatistic):
        return statistic.mean
    elif isinstance(statistic, ProportionStatistic):
        return statistic.proportion
    elif isinstance(statistic, RatioStatistic):
        return statistic.ratio
    elif isinstance(statistic, QuantileStatistic):
        return statistic.mean
    else:
        raise StatisticError(f"Unknown statistic type: {type(statistic)}")


def get_variance(statistic: AnyStatistic) -> float:
    """Extract the variance from any statistic type."""
    return statistic.variance


def get_sample_size(statistic: AnyStatistic) -> int:
    """Extract the sample size from any statistic type."""
    return statistic.n


def validate_statistic_inputs(statistic: AnyStatistic) -> list[str]:
    """
    Comprehensive validation of a statistic object.

    Args:
        statistic: Any statistic object to validate

    Returns:
        List of validation error messages (empty if valid)
    """
    errors = []

    try:
        n = get_sample_size(statistic)
        mean = get_mean(statistic)
        variance = get_variance(statistic)

        # Check for basic validity
        if n <= 0:
            errors.append("Sample size must be positive")

        if not np.isfinite(mean):
            errors.append("Mean must be finite")

        if not np.isfinite(variance) or variance < 0:
            errors.append("Variance must be finite and non-negative")

        # Type-specific validation
        if isinstance(statistic, SampleMeanStatistic):
            errors.extend(_validate_sample_mean(statistic))
        elif isinstance(statistic, ProportionStatistic):
            errors.extend(_validate_proportion(statistic))
        elif isinstance(statistic, QuantileStatistic):
            errors.extend(_validate_quantile(statistic))

    except Exception as e:
        errors.append(f"Failed to validate statistic: {str(e)}")

    return errors


def _validate_sample_mean(stat: SampleMeanStatistic) -> list[str]:
    """Validate SampleMeanStatistic."""
    errors = []

    # Check sum_squares constraint
    if stat.n > 1:
        min_sum_squares = stat.sum**2 / stat.n
        if stat.sum_squares < min_sum_squares - 1e-10:  # Small tolerance for floating point
            errors.append("sum_squares violates mathematical constraint")

    # Check for extreme values
    if abs(stat.sum) > 1e15:
        errors.append("Sum value extremely large - potential overflow risk")

    if stat.sum_squares > 1e30:
        errors.append("Sum of squares extremely large - potential overflow risk")

    return errors


def _validate_proportion(stat: ProportionStatistic) -> list[str]:
    """Validate ProportionStatistic."""
    errors = []

    # Check that sum <= n
    if stat.sum > stat.n:
        errors.append("Number of successes cannot exceed sample size")

    if stat.sum < 0:
        errors.append("Number of successes cannot be negative")

    # Check for extreme proportions
    p = stat.proportion
    if p < 1e-10 or p > 1 - 1e-10:
        errors.append("Proportion extremely close to 0 or 1 - statistical tests may be unreliable")

    return errors


def _validate_quantile(stat: QuantileStatistic) -> list[str]:
    """Validate QuantileStatistic."""
    errors = []

    if stat.has_zero_variance:
        errors.append("Quantile has zero variance - not enough events or too many tied values")

    return errors


def validate_test_inputs(treatment_stat: AnyStatistic, control_stat: AnyStatistic) -> None:
    """
    Comprehensive validation of test inputs.

    Args:
        treatment_stat: Treatment group statistic
        control_stat: Control group statistic

    Raises:
        StatisticError: If validation fails
    """
    # Validate individual statistics
    treatment_errors = validate_statistic_inputs(treatment_stat)
    if treatment_errors:
        raise StatisticError(f"Treatment statistic validation failed: {'; '.join(treatment_errors)}")

    control_errors = validate_statistic_inputs(control_stat)
    if control_errors:
        raise StatisticError(f"Control statistic validation failed: {'; '.join(control_errors)}")

    # Check compatibility
    if type(treatment_stat) is not type(control_stat):
        raise StatisticError("Treatment and control statistics must be the same type")