Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): compute relative effects against the absolute baseline (#127676)

Anders50eb5f669e9c
diff --git a/products/experiments/stats/bayesian/utils.py b/products/experiments/stats/bayesian/utils.py
index d6fb41bf55ea..545fd2967c1b 100644
--- a/products/experiments/stats/bayesian/utils.py
+++ b/products/experiments/stats/bayesian/utils.py
@@ -34,7 +34,7 @@ def calculate_effect_size_and_variance(
Tuple of (effect_size, effect_variance)
Raises:
- StatisticError: If control mean is zero for relative calculations
+ StatisticError: If the baseline mean is zero for relative calculations
"""
treatment_mean = get_mean(treatment_stat)
control_mean = get_mean(control_stat)
@@ -50,13 +50,12 @@ def calculate_effect_size_and_variance(
elif difference_type == DifferenceType.RELATIVE:
denominator = unadjusted_mean if unadjusted_mean is not None else control_mean
- if denominator <= 0:
- raise StatisticError("Control mean must be positive for relative difference calculation")
+ if abs(denominator) < 1e-10:
+ raise StatisticError("Baseline mean is zero, so the relative difference is undefined")
- # Direct relative difference: (μ_T - μ_C) / μ_C
- effect = (treatment_mean - control_mean) / denominator
+ effect = (treatment_mean - control_mean) / abs(denominator)
- # Using delta method for ratios
+ # The effect is ±(μ_T / μ_C - 1), so it has the variance of μ_T / μ_C. Keep the signed μ_C for the covariance term.
effect_variance = variance_of_ratios(
mean_numerator=treatment_mean,
var_numerator=treatment_var / treatment_n,
diff --git a/products/experiments/stats/frequentist/utils.py b/products/experiments/stats/frequentist/utils.py
index 2915e4202d7c..4d2c7f39d765 100644
--- a/products/experiments/stats/frequentist/utils.py
+++ b/products/experiments/stats/frequentist/utils.py
@@ -28,7 +28,7 @@ def calculate_point_estimate(
Point estimate value
Raises:
- StatisticError: If control mean is zero for relative calculations
+ StatisticError: If the baseline mean is zero for relative calculations
"""
treatment_mean = get_mean(treatment_stat)
control_mean = get_mean(control_stat)
@@ -39,8 +39,8 @@ def calculate_point_estimate(
elif difference_type == DifferenceType.RELATIVE:
denominator = unadjusted_mean if unadjusted_mean is not None else control_mean
if abs(denominator) < 1e-10:
- raise StatisticError("Control mean cannot be zero for relative difference calculation")
- return (treatment_mean - control_mean) / denominator
+ raise StatisticError("Baseline mean is zero, so the relative difference is undefined")
+ return (treatment_mean - control_mean) / abs(denominator)
else:
raise StatisticError(f"Unknown difference type: {difference_type}")
@@ -84,7 +84,7 @@ def calculate_variance_pooled(
denominator = unadjusted_mean if unadjusted_mean is not None else control_mean
if abs(denominator) < 1e-10:
- raise StatisticError("Control mean cannot be zero for relative variance calculation")
+ raise StatisticError("Baseline mean is zero, so the relative variance is undefined")
# Var(Y/X) ≈ Var(Y)/X² + Y²Var(X)/X⁴ - 2Y*Cov(X,Y)/X³
# Since treatment and control are independent, Cov(X,Y) = 0
diff --git a/products/experiments/stats/tests/test_bayesian.py b/products/experiments/stats/tests/test_bayesian.py
index e6cbd7a07103..ab46aff58a32 100644
--- a/products/experiments/stats/tests/test_bayesian.py
+++ b/products/experiments/stats/tests/test_bayesian.py
@@ -22,6 +22,8 @@ from products.experiments.stats.bayesian.utils import (
chance_to_win,
credible_interval,
)
+from products.experiments.stats.frequentist.method import FrequentistMethod
+from products.experiments.stats.frequentist.utils import calculate_point_estimate, calculate_variance_pooled
from products.experiments.stats.shared.enums import DifferenceType
from products.experiments.stats.shared.statistics import (
ProportionStatistic,
@@ -432,6 +434,79 @@ class TestEdgeCases:
method.run_test(treatment, control)
+def _sample_mean_stat(n: int, mean: float, variance: float) -> SampleMeanStatistic:
+ total = n * mean
+ return SampleMeanStatistic(n=n, sum=total, sum_squares=(n - 1) * variance + total**2 / n)
+
+
+class TestRelativeEffectSignedBaseline(TestCase):
+ @parameterized.expand(
+ [
+ ("negative_baseline_metric_rises", -10.0, -5.0, 0.5),
+ ("negative_baseline_metric_falls", -8.0, -9.2, -0.15),
+ ("baseline_crosses_zero", -4.0, 2.0, 1.5),
+ ("positive_baseline_metric_rises", 8.0, 9.2, 0.15),
+ ]
+ )
+ def test_relative_effect_is_divided_by_baseline_magnitude(
+ self, _name: str, control_mean: float, treatment_mean: float, expected_relative: float
+ ) -> None:
+ treatment = _sample_mean_stat(n=2000, mean=treatment_mean, variance=36.0)
+ control = _sample_mean_stat(n=2500, mean=control_mean, variance=25.0)
+ expected_variance = 36.0 / 2000 / control_mean**2 + treatment_mean**2 * 25.0 / 2500 / control_mean**4
+
+ frequentist = FrequentistMethod().run_test(treatment, control)
+ bayesian = BayesianMethod().run_test(treatment, control)
+ _, bayesian_variance = calculate_effect_size_and_variance(treatment, control, DifferenceType.RELATIVE)
+
+ assert frequentist.point_estimate == pytest.approx(expected_relative)
+ assert bayesian.effect_size == pytest.approx(expected_relative)
+ assert calculate_variance_pooled(treatment, control, DifferenceType.RELATIVE) == pytest.approx(
+ expected_variance
+ )
+ assert bayesian_variance == pytest.approx(expected_variance)
+ assert (bayesian.chance_to_win > 0.5) == (expected_relative > 0)
+ assert (frequentist.confidence_interval[0] > 0) == (expected_relative > 0)
+
+ def test_negating_both_arms_flips_effects_and_keeps_variance(self) -> None:
+ treatment = _sample_mean_stat(n=1500, mean=-6.0, variance=16.0)
+ control = _sample_mean_stat(n=1500, mean=-7.5, variance=20.0)
+ negated_treatment = _sample_mean_stat(n=1500, mean=6.0, variance=16.0)
+ negated_control = _sample_mean_stat(n=1500, mean=7.5, variance=20.0)
+
+ for difference_type, expected in [(DifferenceType.ABSOLUTE, 1.5), (DifferenceType.RELATIVE, 1.5 / 7.5)]:
+ effect = calculate_point_estimate(treatment, control, difference_type)
+ negated_effect = calculate_point_estimate(negated_treatment, negated_control, difference_type)
+ bayesian_effect, bayesian_variance = calculate_effect_size_and_variance(treatment, control, difference_type)
+ negated_bayesian_effect, negated_bayesian_variance = calculate_effect_size_and_variance(
+ negated_treatment, negated_control, difference_type
+ )
+
+ assert effect == pytest.approx(expected)
+ assert negated_effect == pytest.approx(-expected)
+ assert bayesian_effect == pytest.approx(expected)
+ assert negated_bayesian_effect == pytest.approx(-expected)
+ assert calculate_variance_pooled(treatment, control, difference_type) == pytest.approx(
+ calculate_variance_pooled(negated_treatment, negated_control, difference_type)
+ )
+ assert bayesian_variance == pytest.approx(negated_bayesian_variance)
+
+ @parameterized.expand(
+ [
+ ("frequentist", 0.0, lambda t, c: FrequentistMethod().run_test(t, c)),
+ ("bayesian", 0.0, lambda t, c: BayesianMethod().run_test(t, c)),
+ ("frequentist_cuped_baseline", 2.0, lambda t, c: FrequentistMethod().run_test(t, c, unadjusted_mean=0.0)),
+ ("bayesian_cuped_baseline", 2.0, lambda t, c: BayesianMethod().run_test(t, c, unadjusted_mean=1e-12)),
+ ]
+ )
+ def test_zero_baseline_is_rejected(self, _name: str, control_mean: float, run_test: Any) -> None:
+ treatment = _sample_mean_stat(n=500, mean=3.0, variance=4.0)
+ control = _sample_mean_stat(n=500, mean=control_mean, variance=4.0)
+
+ with pytest.raises(StatisticError, match="Baseline mean is zero"):
+ run_test(treatment, control)
+
+
class TestNumericalStability:
"""Tests for numerical stability and precision."""