Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): name more causes of a sample ratio mismatch (#127851)

Marcel Poelkerfaa4d603dd6f
diff --git a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx
index 9ef683aa39b1..e3c36cb5a044 100644
--- a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx
@@ -20,7 +20,7 @@ import { ExperimentExposureCriteria, ExperimentExposureQueryResponse } from '~/q
import { EXPERIMENT_VARIANT_MULTIPLE } from 'products/experiments/frontend/constants'
import {
getTotalExposures,
- hasSampleRatioMismatch,
+ reportsMismatch,
sampleRatioLine,
serverSampleRatioFinding,
} from 'products/experiments/frontend/health/exposureHealth'
@@ -37,6 +37,11 @@ import { VariantTag } from './VariantTag'
// from a stuck one, so we start showing elapsed time and a way to retry.
const SLOW_LOAD_THRESHOLD_SECONDS = 20
+const MISMATCH_LINE_STYLE = {
+ warning: { className: 'text-warning font-semibold', icon: <IconWarning className="text-sm" /> },
+ info: { className: 'text-secondary', icon: <IconInfo className="text-sm" /> },
+}
+
/** Seconds since `active` last became true, ticking every second, reset to 0 when it goes false. */
function useElapsedSeconds(active: boolean): number {
const [elapsed, setElapsed] = useState(0)
@@ -167,9 +172,8 @@ export function Exposures(): JSX.Element {
const multipleTreatmentLabel =
resolvedHandling === 'first_seen' ? 'using first seen variant' : 'excluded from analysis'
- // Detect sample ratio mismatch (p < 0.001 is significant)
- const hasSRM = hasSampleRatioMismatch(exposures)
const srmLine = sampleRatioLine(exposures, healthUi === 'panel')
+ const hasSRM = reportsMismatch(srmLine)
// The open panel says "No exposures yet" for a draft and after a failed exposure query too.
// Only a launched experiment with an answer that holds no exposure is the zero-exposure state.
@@ -496,12 +500,14 @@ export function Exposures(): JSX.Element {
/>
{srmLine && exposures?.sample_ratio_mismatch != null && (
<div className="flex items-center gap-1 text-xs mt-2">
- {srmLine.tone === 'warning' ? (
+ {reportsMismatch(srmLine) ? (
<>
<Tooltip title={srmLine.tooltip}>
- <span className="flex items-center gap-1 text-warning cursor-pointer">
- <IconWarning className="text-sm" />
- <span className="font-semibold">{srmLine.label}</span>
+ <span
+ className={`flex items-center gap-1 cursor-pointer ${MISMATCH_LINE_STYLE[srmLine.tone].className}`}
+ >
+ {MISMATCH_LINE_STYLE[srmLine.tone].icon}
+ <span>{srmLine.label}</span>
</span>
</Tooltip>
<span className="text-muted">
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index 89831703d782..75b03bb25960 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -60,6 +60,13 @@ class _Pins:
graded: PinnedConditions
+def assigns_nobody_by_split(flag: FlagState) -> bool:
+ """Whether every release condition that includes users sets a variant override, so the split assigns nobody."""
+ return bool(flag.reachable_conditions) and all(
+ condition.pinned_variant is not None for condition in flag.reachable_conditions
+ )
+
+
def overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
"""The overrides in place that SOME_CONDITIONS_PINNED is graded by, or None when it is not graded."""
flag = flag_to_check(ctx, before_launch=True)
@@ -121,7 +128,7 @@ def _pins(ctx: HealthContext, flag: FlagState) -> _Pins | None:
analyzed = tuple(condition for condition in pinned if condition.pinned_variant not in ctx.excluded_variant_keys)
return _Pins(
pinned=pinned,
- all_pinned=len(pinned) == len(flag.reachable_conditions),
+ all_pinned=assigns_nobody_by_split(flag),
analyzed=analyzed,
graded=pinned_conditions(analyzed, flag.variants),
)
@@ -191,7 +198,7 @@ def _set_after_launch(ctx: HealthContext, numbers: Sequence[int]) -> _SetAfterLa
"who were exposed before may have switched variants."
)
else:
- names = _set_names(sorted(override.condition_number for override in added))
+ names = set_names(sorted(override.condition_number for override in added))
sentence = (
f" The overrides of {names} were set after launch, last on {at}, so users they match who were exposed "
"before may have switched variants."
@@ -212,7 +219,7 @@ def _quoted(keys: Sequence[str]) -> str:
return ", ".join(f'"{key}"' for key in keys)
-def _set_names(numbers: Sequence[int]) -> str:
+def set_names(numbers: Sequence[int]) -> str:
if len(numbers) == 1:
return f"Set {numbers[0]}"
return "Sets " + ", ".join(str(number) for number in numbers[:-1]) + f" and {numbers[-1]}"
@@ -275,13 +282,13 @@ def _some_pinned_detail(
key = pinned_keys[0]
overrides, them = "the override", "it"
fact = (
- f'{_set_names(numbers)} sets the variant override "{key}", so users who match it always get "{key}" '
+ f'{set_names(numbers)} sets the variant override "{key}", so users who match it always get "{key}" '
"instead of a random variant."
)
else:
overrides, them = "the overrides", "them"
fact = (
- f"{_set_names(numbers)} set variant overrides ({_quoted(pinned_keys)}), so users who match them "
+ f"{set_names(numbers)} set variant overrides ({_quoted(pinned_keys)}), so users who match them "
"always get the variant of their condition instead of a random one."
)
why = (
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index 391a203a3299..dfc1d995b324 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -1,3 +1,4 @@
+import math
import unicodedata
from collections.abc import Mapping
from datetime import date
@@ -17,14 +18,30 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
)
+from products.experiments.backend.health.checks.forced_variant import (
+ SOME_CONDITIONS_PINNED,
+ assigns_nobody_by_split,
+ set_names,
+)
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, FlagHistory, HealthContext
+from products.experiments.backend.health.context import (
+ ExposureSegment,
+ ExposureTotals,
+ FlagHistory,
+ FlagState,
+ HealthContext,
+ ReachableCondition,
+)
+from products.experiments.backend.health.pinned_share import pinned_variant_keys
from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.feature_flags.backend.facade.edits import SplitChange
SAMPLE_RATIO_MISMATCH_P_VALUE = 0.001
SRM_MINIMUM_SAMPLE_SIZE = 100
+# The share of persons or groups that would have to switch variants to match the split. Below it, a mismatch moves too
+# few of them to change the results much, so it is info. A large experiment reaches the p-value with a far smaller skew.
+MATERIAL_SHARE_OFF_SPLIT = 0.01
TOP_SEGMENT_VALUES = 5
# Below this, a skew of a few points passes the homogeneity test for lack of power, and would read as the same skew
# everywhere. Two segments this large are needed before the finding calls a dimension uniform.
@@ -91,6 +108,7 @@ def reads_segments(ctx: HealthContext) -> bool:
outcome = srm(ctx)
if not isinstance(outcome, ExperimentHealthFinding):
return False
+ # A variant override can explain a mismatch, but a segment that locates it names the cause more precisely.
if outcome.subcode != SPLIT_CHANGED:
return True
retest_p_value = outcome.evidence.get("p_value_after_split_change")
@@ -102,7 +120,7 @@ def segments_first_exposed_after(ctx: HealthContext) -> date | None:
change, the users first exposed before it got their variants from another split, so they would fail the test
against the current split in any segment they fill."""
history = ctx.flag_history
- if not isinstance(history, FlagHistory) or _split_change_warning(ctx) is None:
+ if not isinstance(history, FlagHistory) or _split_change_cause(ctx) is None:
return None
return _last_split_change_day(history)
@@ -111,6 +129,10 @@ def srm(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None:
return not_applicable(RUNS_AFTER_LAUNCH)
+ # The split then serves nobody, so the test compares nothing. `forced_variant_release_condition` explains it.
+ overrides = _flag_of_the_run(ctx)
+ if overrides is not None and assigns_nobody_by_split(overrides):
+ return not_applicable("The split assigns nobody, because every release condition sets a variant override")
p_value = exposures.sample_ratio_mismatch_p_value
if p_value is None:
# The exposure query compares only the variants with a share of the split that the analysis keeps, so after
@@ -125,16 +147,55 @@ def srm(ctx: HealthContext) -> CheckOutcome:
return not_applicable("Runs once enough users are exposed")
if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
return None
+ share_off_split = _share_off_split(exposures.total_exposures, exposures.sample_ratio_expected)
split_change = _split_change_warning(ctx)
history = ctx.flag_history
- if split_change is None or not isinstance(history, FlagHistory):
- return _mismatch(exposures, p_value)
- # The warning stays a warning: the users exposed after the change are fewer than those the test covers, so a
- # retest that passes can't rule out a mismatch with another cause. The retest only adds evidence.
+ ruled_out = False
+ if split_change is not None and isinstance(history, FlagHistory):
+ if not _skew_predates_the_change(ctx, exposures, history):
+ return _after_split_change(ctx, exposures, history, split_change, p_value, share_off_split)
+ ruled_out = True
+ cause = _segment_cause(exposures, exposures.unit)
+ pinned = _pinned_cause(ctx, exposures)
+ if pinned and not cause.locates_the_mismatch:
+ return _pinned_mismatch(exposures, pinned, p_value, share_off_split, cause)
+ return _mismatch(exposures, p_value, share_off_split, cause, split_change_ruled_out=ruled_out)
+
+
+def _severity(share_off_split: float) -> ExperimentHealthFindingSeverity:
+ if share_off_split < MATERIAL_SHARE_OFF_SPLIT:
+ return ExperimentHealthFindingSeverity.INFO
+ return ExperimentHealthFindingSeverity.WARNING
+
+
+def _small_mismatch(share_off_split: float, unit: str) -> list[str]:
+ if share_off_split >= MATERIAL_SHARE_OFF_SPLIT:
+ return []
+ # Rounded down, so that a share just under the floor doesn't read as the floor. The epsilon keeps a share of 0.008
+ # that float arithmetic computes as 0.00799… at 0.8%.
+ tenths = math.floor(share_off_split * 1000 + 1e-9)
+ share = f"Only {tenths / 10:g}%" if tenths else "Fewer than 0.1%"
+ return [
+ f"{share} of {unit}s would have to switch variants to match the split, so the difference is unlikely to "
+ "change the results."
+ ]
+
+
+def _after_split_change(
+ ctx: HealthContext,
+ exposures: ExposureTotals,
+ history: FlagHistory,
+ split_change: ExperimentHealthFinding,
+ p_value: float,
+ share_off_split: float,
+) -> ExperimentHealthFinding:
+ # A retest that passes keeps the severity: the users exposed after the change are fewer than those the test covers,
+ # so it can't rule out a mismatch with another cause. The retest only adds evidence.
unit = exposures.unit
retest, p_value_after_change = _retest_after_change(ctx, exposures, history, unit)
evidence: dict[str, str | int | float | None] = {
"p_value": p_value,
+ "share_off_split": round(share_off_split, 4),
"p_value_after_split_change": p_value_after_change,
}
cause = (
@@ -148,12 +209,13 @@ def srm(ctx: HealthContext) -> CheckOutcome:
code=ExperimentHealthFindingCode.SRM,
subcode=SPLIT_CHANGED,
reason=cause.reason if cause is not None else None,
- severity=ExperimentHealthFindingSeverity.WARNING,
+ severity=_severity(share_off_split),
title=f"{unit.capitalize()}s are not split as configured after a split change",
detail=" ".join(
[
f"The distribution of {unit}s across variants doesn't match your current rollout percentages "
f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",
+ *_small_mismatch(share_off_split, unit),
f"The variant split changed after launch while {unit}s were exposed, and some of them got their "
"variants from another split, which can cause this.",
*retest,
@@ -207,6 +269,49 @@ def _last_split_change_day(history: FlagHistory) -> date:
return history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()
+def _split_change_cause(ctx: HealthContext) -> ExperimentHealthFinding | None:
+ """The warning of the split change check, when the change can explain the mismatch."""
+ warning = _split_change_warning(ctx)
+ history = ctx.flag_history
+ if warning is None or not isinstance(history, FlagHistory) or ctx.exposures is None:
+ return None
+ return None if _skew_predates_the_change(ctx, ctx.exposures, history) else warning
+
+
+def _skew_predates_the_change(ctx: HealthContext, exposures: ExposureTotals, history: FlagHistory) -> bool:
+ """Whether the users first exposed before the first split change, those first exposed after the last one and all
+ exposed users show the same shares, and those after it don't match the current split. Mixing assignments then
+ can't explain the mismatch, because each group of users alone shows it."""
+ events = ctx.exposure_events
+ if events is None or events.activation is not None:
+ return False
+ changes = history.counted(SplitChange)
+ if not changes:
+ return False
+ first_day = changes[0].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()
+ last_day = _last_split_change_day(history)
+ weights = exposures.sample_ratio_expected
+ # A user first exposed on the day of a change can have either split, so neither group counts them.
+ before = {
+ variant: sum(count for day, count in exposures.daily_exposures.get(variant, {}).items() if day < first_day)
+ for variant in weights
+ }
+ after = {
+ variant: sum(count for day, count in exposures.daily_exposures.get(variant, {}).items() if day > last_day)
+ for variant in weights
+ }
+ retest = sample_ratio_test(after, weights)
+ if retest is None or retest.p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
+ return False
+ if sum(before.values()) < SRM_MINIMUM_SAMPLE_SIZE:
+ return False
+ # The users between the changes and on a change day fill neither group, so all users must show the shares too.
+ return (
+ _share_off_split(before, after) < MATERIAL_SHARE_OFF_SPLIT
+ and _share_off_split(exposures.total_exposures, after) < MATERIAL_SHARE_OFF_SPLIT
+ )
+
+
def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:
try:
outcome = split_changed_mid_run(ctx)
@@ -219,23 +324,113 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:
return None
-def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:
+def _flag_of_the_run(ctx: HealthContext) -> FlagState | None:
+ """The flag whose variant overrides can explain the mismatch. None after the end, because the flag then often
+ holds the shipped decision and not the setup of the run, like `flag_to_check` says."""
+ return None if ctx.has_ended else ctx.flag
+
+
+def _pinned_cause(ctx: HealthContext, exposures: ExposureTotals) -> tuple[ReachableCondition, ...]:
+ """The release conditions with a variant override in the test, when one of them pins the variant furthest above
+ its expected count. Their users don't follow the split, so they can cause the mismatch."""
+ expected = exposures.sample_ratio_expected
+ flag = _flag_of_the_run(ctx)
+ if flag is None or not expected:
+ return ()
+ pinned = tuple(condition for condition in flag.reachable_conditions if condition.pinned_variant in expected)
+ over = max(expected, key=lambda variant: exposures.total_exposures.get(variant, 0) - expected[variant])
+ return pinned if any(condition.pinned_variant == over for condition in pinned) else ()
+
+
+def _pinned_mismatch(
+ exposures: ExposureTotals,
+ pinned: tuple[ReachableCondition, ...],
+ p_value: float,
+ share_off_split: float,
+ segments: "_SegmentCause",
+) -> ExperimentHealthFinding:
unit = exposures.unit
- cause = _segment_cause(exposures, unit)
+ numbers = [condition.number for condition in pinned]
+ keys = pinned_variant_keys(pinned)
+ if len(numbers) == 1:
+ cause = (
+ f'{set_names(numbers)} sets the variant override "{keys[0]}", so the {unit}s it matches get "{keys[0]}" '
+ "instead of a variant from the split, which can cause this."
+ )
+ overrides = "the override"
+ else:
+ quoted = ", ".join(f'"{key}"' for key in keys)
+ cause = (
+ f"{set_names(numbers)} set variant overrides ({quoted}), so the {unit}s they match get the variant of "
+ "their condition instead of one from the split, which can cause this."
+ )
+ overrides = "the overrides"
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.SRM,
+ subcode=SOME_CONDITIONS_PINNED,
+ reason=None,
+ severity=_severity(share_off_split),
+ title=f"{unit.capitalize()}s are not split as configured because of variant overrides",
+ detail=" ".join(
+ [
+ f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "
+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",
+ *_small_mismatch(share_off_split, unit),
+ cause,
+ "If these are internal or test users, check that the filter for internal and test users excludes "
+ f"them from the results. Otherwise remove {overrides}.",
+ ]
+ ),
+ evidence={
+ "p_value": p_value,
+ "share_off_split": round(share_off_split, 4),
+ "pinned_conditions": ",".join(str(number) for number in numbers),
+ "pinned_variants": ",".join(keys),
+ **segments.evidence,
+ },
+ actions=(),
+ diagnostic_ref="A2",
+ )
+
+
+def _mismatch(
+ exposures: ExposureTotals,
+ p_value: float,
+ share_off_split: float,
+ cause: "_SegmentCause",
+ *,
+ split_change_ruled_out: bool,
+) -> ExperimentHealthFinding:
+ unit = exposures.unit
+ ruled_out = (
+ [
+ f"The variant split changed after launch, but the {unit}s first exposed before the change show the same "
+ "mismatch as those first exposed after it, so the change doesn't explain it."
+ ]
+ if split_change_ruled_out
+ else []
+ )
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.SRM,
subcode=None,
reason=cause.reason,
- severity=ExperimentHealthFindingSeverity.WARNING,
+ severity=_severity(share_off_split),
title=f"{unit.capitalize()}s are not split across variants as configured",
detail=" ".join(
[
f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "
f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",
+ *_small_mismatch(share_off_split, unit),
+ *ruled_out,
*(cause.sentences or ("This may indicate issues with randomization or data collection.",)),
]
),
- evidence={"p_value": p_value, **cause.evidence},
+ evidence={
+ "p_value": p_value,
+ "share_off_split": round(share_off_split, 4),
+ **({"split_change_ruled_out": True} if split_change_ruled_out else {}),
+ **cause.evidence,
+ },
actions=(),
diagnostic_ref="A2",
)
@@ -266,6 +461,10 @@ class _SegmentCause:
reason: str | None
evidence: dict[str, str | int | float | None]
+ @property
+ def locates_the_mismatch(self) -> bool:
+ return self.reason is not None and self.reason != SrmReason.UNIFORM
+
def _segment_cause(exposures: ExposureTotals, unit: str) -> _SegmentCause:
tests = _segment_tests(exposures)
@@ -314,17 +513,19 @@ def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, wei
homogeneity_p_value=homogeneity_p_value,
powered=powered,
located=located,
- distance=_distance_from_split(located, weights) if located is not None else 0.0,
+ distance=_share_off_split(located.exposed_by_variant, weights) if located is not None else 0.0,
)
-def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:
- """The share of the segment's persons or groups that would have to move between variants to match the split."""
- total = sum(segment.exposed_by_variant.get(variant, 0) for variant in weights)
+def _share_off_split(exposed_by_variant: Mapping[str, int], weights: Mapping[str, float]) -> float:
+ """The share of the persons or groups that would have to move between variants to match the weights."""
+ total = sum(exposed_by_variant.get(variant, 0) for variant in weights)
total_weight = sum(weights.values())
+ if total <= 0 or total_weight <= 0:
+ return 0.0
return (
sum(
- abs(segment.exposed_by_variant.get(variant, 0) / total - weight / total_weight)
+ abs(exposed_by_variant.get(variant, 0) / total - weight / total_weight)
for variant, weight in weights.items()
)
/ 2
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index b2d1e15445fe..d668a72166fc 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -134,7 +134,8 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
# The exposure query runs these on its answer. They must not include the checks above: the exposure answer
# is cached for a day, and a flag edit would not clear their findings. `srm` reads the verdict of
-# `split_changed_mid_run`, which only a split edit can change, and the split is part of the exposure query's cache key.
+# `split_changed_mid_run`, which only a split edit can change, and the variant overrides of the release conditions. The
+# split and the release conditions are part of the exposure query's cache key.
EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.ZERO_EXPOSURES,
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index edf68d62901f..6c52472fb08f 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -664,6 +664,63 @@ class TestSrmAfterSplitChange(TestCase):
self.assertEqual((reads_flag_history(ctx), reads_segments(ctx), segments_first_exposed_after(ctx)), expected)
+ @parameterized.expand(
+ [
+ (
+ "same_skew_before_the_change",
+ {"control": 600, "test": 400},
+ {"control": 0, "test": 0},
+ (None, True, None),
+ "The variant split changed after launch, but the users first exposed before the change show the same "
+ "mismatch as those first exposed after it, so the change doesn't explain it.",
+ ),
+ (
+ "another_skew_before_the_change",
+ {"control": 500, "test": 500},
+ {"control": 0, "test": 0},
+ ("split_changed", True, date(2026, 9, 14)),
+ "don't match the current split either (p < 0.001), so look for another cause too.",
+ ),
+ (
+ "another_skew_on_the_change_day",
+ {"control": 600, "test": 400},
+ {"control": 5000, "test": 0},
+ ("split_changed", True, date(2026, 9, 14)),
+ "don't match the current split either (p < 0.001), so look for another cause too.",
+ ),
+ ]
+ )
+ def test_rules_out_a_split_change_when_the_skew_predates_it(
+ self,
+ _name: str,
+ before_change: dict[str, int],
+ on_change_day: dict[str, int],
+ expected: tuple[str | None, bool, date | None],
+ text: str,
+ ) -> None:
+ after_change = {"control": 600, "test": 400}
+ days = {date(2026, 9, 12): before_change, date(2026, 9, 14): on_change_day, date(2026, 9, 15): after_change}
+ total_exposures = {variant: sum(day[variant] for day in days.values()) for variant in after_change}
+ whole_run = sum(total_exposures.values())
+ totals = replace(
+ _totals(after_change),
+ total_exposures=total_exposures,
+ sample_ratio_expected={"control": 0.8 * whole_run, "test": 0.2 * whole_run},
+ daily_exposures={
+ variant: {day: day_counts[variant] for day, day_counts in days.items()} for variant in after_change
+ },
+ )
+ ctx = replace(
+ _context(EVEN, UNEVEN, exposed_units=5, first_change_at=datetime(2026, 9, 14, 13, 4, tzinfo=UTC)),
+ exposures=totals,
+ )
+
+ outcome = srm(ctx)
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual((outcome.subcode, reads_segments(ctx), segments_first_exposed_after(ctx)), expected)
+ self.assertIn(text, outcome.detail)
+
def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:
outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))
diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.py
index 06687664b0f7..861bd9602317 100644
--- a/products/experiments/backend/health/test/test_srm.py
+++ b/products/experiments/backend/health/test/test_srm.py
@@ -8,9 +8,10 @@ from posthog.schema import MultipleVariantHandling
from posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL
-from products.experiments.backend.facade.contracts import ExperimentHealthFinding
-from products.experiments.backend.health.checks.srm import srm
-from products.experiments.backend.health.context import ExposureSegment, ExposureTotals
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverity
+from products.experiments.backend.health.checks.srm import reads_segments, srm
+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, parse_flag_state
+from products.experiments.backend.health.skips import CheckSkip
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
@@ -211,6 +212,7 @@ class TestSrmSegments(TestCase):
evidence,
{
"p_value": 0.00001,
+ "share_off_split": 0.1,
"located_dimensions": "library",
"library_1": "web",
"library_1_p_value": 1.0,
@@ -230,3 +232,169 @@ class TestSrmSegments(TestCase):
"page_2_exposed_test": 200,
},
)
+
+
+EVEN_SPLIT = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
+EVERYONE = {"properties": [], "rollout_percentage": 100}
+
+
+PINNED_TEST = (
+ f'{MISMATCH} Set 1 sets the variant override "test", so the users it matches get "test" instead of a variant from '
+ "the split, which can cause this. If these are internal or test users, check that the filter for internal and test "
+ "users excludes them from the results. Otherwise remove the override."
+)
+
+
+def _pinned_to(variant: str) -> dict[str, object]:
+ return {
+ "properties": [{"key": "$geoip_country_code", "value": ["DE"], "operator": "exact", "type": "person"}],
+ "rollout_percentage": 100,
+ "variant": variant,
+ }
+
+
+class TestSrmCauses(TestCase):
+ @parameterized.expand(
+ [
+ (
+ "half_a_percent_off",
+ {"control": 50500, "test": 49500},
+ {"control": 50000, "test": 50000},
+ ExperimentHealthFindingSeverity.INFO,
+ f"{MISMATCH} Only 0.5% of users would have to switch variants to match the split, so the difference "
+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",
+ ),
+ (
+ "three_variants_under_one_percent_off",
+ {"control": 33200, "test": 33300, "blue": 33500},
+ {"control": 34000, "test": 33000, "blue": 33000},
+ ExperimentHealthFindingSeverity.INFO,
+ f"{MISMATCH} Only 0.8% of users would have to switch variants to match the split, so the difference "
+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",
+ ),
+ (
+ "just_under_one_percent_off",
+ {"control": 50990, "test": 49010},
+ {"control": 50000, "test": 50000},
+ ExperimentHealthFindingSeverity.INFO,
+ f"{MISMATCH} Only 0.9% of users would have to switch variants to match the split, so the difference "
+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",
+ ),
+ (
+ "under_a_tenth_of_a_percent_off",
+ {"control": 5002000, "test": 4998000},
+ {"control": 5000000, "test": 5000000},
+ ExperimentHealthFindingSeverity.INFO,
+ f"{MISMATCH} Fewer than 0.1% of users would have to switch variants to match the split, so the "
+ "difference is unlikely to change the results. This may indicate issues with randomization or data "
+ "collection.",
+ ),
+ (
+ "one_percent_off",
+ {"control": 51000, "test": 49000},
+ {"control": 50000, "test": 50000},
+ ExperimentHealthFindingSeverity.WARNING,
+ NO_LOCATION,
+ ),
+ ]
+ )
+ def test_a_mismatch_of_under_one_percent_of_users_is_info(
+ self,
+ _name: str,
+ total_exposures: dict[str, int],
+ expected: dict[str, float],
+ severity: ExperimentHealthFindingSeverity,
+ detail: str,
+ ) -> None:
+ exposures = replace(_exposures(None), total_exposures=total_exposures, sample_ratio_expected=expected)
+
+ outcome = srm(replace(RUNNING_CONTEXT, exposures=exposures))
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual((outcome.severity, outcome.detail), (severity, detail))
+
+ @parameterized.expand(
+ [
+ (
+ "override_of_the_variant_in_excess",
+ [_pinned_to("test"), EVERYONE],
+ False,
+ None,
+ ("some_conditions_pinned", True),
+ PINNED_TEST,
+ ),
+ (
+ "override_and_a_uniform_mismatch",
+ [_pinned_to("test"), EVERYONE],
+ False,
+ (_segment(LIBRARY, "web", 400, 600), _segment(LIBRARY, "posthog-node", 400, 600)),
+ ("some_conditions_pinned", True),
+ PINNED_TEST,
+ ),
+ (
+ "override_and_a_segment_with_the_mismatch",
+ [_pinned_to("test"), EVERYONE],
+ False,
+ SERVER_SKEWED,
+ (None, True),
+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by the "posthog-node" '
+ f"library. {NEXT_STEP}",
+ ),
+ (
+ "override_of_the_other_variant",
+ [_pinned_to("control"), EVERYONE],
+ False,
+ None,
+ (None, True),
+ NO_LOCATION,
+ ),
+ ("override_after_the_end", [_pinned_to("test"), EVERYONE], True, None, (None, True), NO_LOCATION),
+ (
+ "every_condition_overridden_after_the_end",
+ [_pinned_to("test"), {**EVERYONE, "variant": "control"}],
+ True,
+ None,
+ (None, True),
+ NO_LOCATION,
+ ),
+ ]
+ )
+ def test_names_a_variant_override_of_the_variant_in_excess(
+ self,
+ _name: str,
+ groups: list[dict[str, object]],
+ has_ended: bool,
+ segments: tuple[ExposureSegment, ...] | None,
+ expected: tuple[str | None, bool],
+ detail: str,
+ ) -> None:
+ ctx = replace(
+ RUNNING_CONTEXT,
+ has_ended=has_ended,
+ flag=parse_flag_state(active=True, deleted=False, groups=groups, variants=EVEN_SPLIT),
+ exposures=_exposures(segments),
+ )
+
+ outcome = srm(ctx)
+
+ assert isinstance(outcome, ExperimentHealthFinding)
+ self.assertEqual(((outcome.subcode, reads_segments(ctx)), outcome.detail), (expected, detail))
+
+ def test_leaves_a_split_that_assigns_nobody_to_the_variant_override_finding(self) -> None:
+ ctx = replace(
+ RUNNING_CONTEXT,
+ flag=parse_flag_state(
+ active=True,
+ deleted=False,
+ groups=[_pinned_to("test"), {**EVERYONE, "variant": "control"}],
+ variants=EVEN_SPLIT,
+ ),
+ exposures=_exposures(None),
+ )
+
+ outcome = srm(ctx)
+
+ assert isinstance(outcome, CheckSkip)
+ self.assertEqual(
+ outcome.reason, "The split assigns nobody, because every release condition sets a variant override"
+ )
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index 1d23db7c25a1..4a30ba41d3c7 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -1746,6 +1746,7 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
self.assertEqual(
{key: value for key, value in finding.evidence.items() if not key.endswith("p_value")},
{
+ "share_off_split": 0.1667,
"located_dimensions": "library,library_version",
"library_1": "web",
"library_1_exposed_control": 100,
@@ -1934,7 +1935,7 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
ExperimentExposureHealthFindingCode.SRM,
"The distribution of users across variants doesn't match your configured rollout percentages "
"(p < 0.001). This may indicate issues with randomization or data collection.",
- {"p_value"},
+ {"p_value", "share_off_split"},
)
],
)
diff --git a/products/experiments/frontend/health/exposureHealth.test.ts b/products/experiments/frontend/health/exposureHealth.test.ts
index 3c1c49fb7b87..015d9339e3c1 100644
--- a/products/experiments/frontend/health/exposureHealth.test.ts
+++ b/products/experiments/frontend/health/exposureHealth.test.ts
@@ -1,14 +1,19 @@
-import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'
+import {
+ type ExperimentExposureHealthCheck,
+ type ExperimentExposureQueryResponse,
+ NodeKind,
+} from '~/queries/schema/schema-general'
import { SAMPLE_RATIO_MISMATCH_DESCRIPTION, sampleRatioLine } from './exposureHealth'
const srmFinding = (
- subcode: string | null
+ subcode: string | null,
+ severity: 'warning' | 'info' = 'warning'
): NonNullable<ExperimentExposureQueryResponse['health_findings']>[number] => ({
code: 'srm',
subcode,
reason: null,
- severity: 'warning',
+ severity,
title: 'Users are not split as configured',
detail: `Server detail for ${subcode ?? 'a mismatch'}`,
evidence: {},
@@ -26,6 +31,13 @@ const MISMATCH = {
health_checks: [],
} satisfies ExperimentExposureQueryResponse
+const SRM_NOT_APPLICABLE: ExperimentExposureHealthCheck = {
+ key: 'srm',
+ name: 'Sample ratio mismatch',
+ status: 'not_applicable',
+ reason: 'The split assigns nobody, because every release condition sets a variant override',
+}
+
describe('sampleRatioLine', () => {
it.each([
{
@@ -48,6 +60,18 @@ describe('sampleRatioLine', () => {
tooltip: 'Server detail for a mismatch',
},
},
+ {
+ desc: 'a mismatch the server reports as info',
+ exposures: { ...MISMATCH, health_findings: [srmFinding(null, 'info')] },
+ followsServer: true,
+ expected: { tone: 'info', label: 'Sample ratio mismatch detected' },
+ },
+ {
+ desc: 'a split that every variant override bypasses',
+ exposures: { ...MISMATCH, health_findings: [], health_checks: [SRM_NOT_APPLICABLE] },
+ followsServer: true,
+ expected: null,
+ },
{
desc: 'a reader without the health panel',
exposures: MISMATCH,
@@ -69,6 +93,6 @@ describe('sampleRatioLine', () => {
},
},
])('follows the server only with the health panel: $desc', ({ exposures, followsServer, expected }) => {
- expect(sampleRatioLine(exposures, followsServer)).toMatchObject(expected)
+ expect(sampleRatioLine(exposures, followsServer)).toEqual(expected && expect.objectContaining(expected))
})
})
diff --git a/products/experiments/frontend/health/exposureHealth.ts b/products/experiments/frontend/health/exposureHealth.ts
index 4de5e21f764a..3b96aa23efee 100644
--- a/products/experiments/frontend/health/exposureHealth.ts
+++ b/products/experiments/frontend/health/exposureHealth.ts
@@ -24,11 +24,16 @@ export function serverSampleRatioFinding(
/** What the exposures card says about the sample ratio test. */
export type SampleRatioLine = {
- tone: 'warning' | 'success'
+ tone: 'warning' | 'info' | 'success'
label: string
tooltip: string
}
+/** Whether the exposures card's line reports a mismatch, as a warning or as info. */
+export function reportsMismatch(line: SampleRatioLine | null): line is SampleRatioLine & { tone: 'warning' | 'info' } {
+ return line !== null && line.tone !== 'success'
+}
+
/**
* The exposures card's line about the sample ratio test, or null without a test. With the health panel the card uses
* the server's finding, which can name a split change as the likely cause. Without it the card keeps the page's own
@@ -41,10 +46,17 @@ export function sampleRatioLine(
if (exposures?.sample_ratio_mismatch == null) {
return null
}
+ // SRM doesn't apply when every release condition sets a variant override. The override finding explains it.
+ if (
+ followsServer &&
+ exposures.health_checks?.some(({ key, status }) => key === 'srm' && status === 'not_applicable')
+ ) {
+ return null
+ }
const finding = followsServer ? serverSampleRatioFinding(exposures) : undefined
if (finding) {
return {
- tone: 'warning',
+ tone: finding.severity === 'info' ? 'info' : 'warning',
label:
finding.subcode === 'split_changed'
? 'Sample ratio mismatch after a split change'