:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): name more causes of a sample ratio mismatch (#127851)
Marcel Poelkerfaa4d603dd6f
diff --git a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsxindex 9ef683aa39b1..e3c36cb5a044 100644--- a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx@@ -20,7 +20,7 @@ import { ExperimentExposureCriteria, ExperimentExposureQueryResponse } from '~/qimport { EXPERIMENT_VARIANT_MULTIPLE } from 'products/experiments/frontend/constants'import {getTotalExposures,- hasSampleRatioMismatch,+ reportsMismatch,sampleRatioLine,serverSampleRatioFinding,} from 'products/experiments/frontend/health/exposureHealth'@@ -37,6 +37,11 @@ import { VariantTag } from './VariantTag'// from a stuck one, so we start showing elapsed time and a way to retry.const SLOW_LOAD_THRESHOLD_SECONDS = 20+const MISMATCH_LINE_STYLE = {+ warning: { className: 'text-warning font-semibold', icon: <IconWarning className="text-sm" /> },+ info: { className: 'text-secondary', icon: <IconInfo className="text-sm" /> },+}+/** Seconds since `active` last became true, ticking every second, reset to 0 when it goes false. */function useElapsedSeconds(active: boolean): number {const [elapsed, setElapsed] = useState(0)@@ -167,9 +172,8 @@ export function Exposures(): JSX.Element {const multipleTreatmentLabel =resolvedHandling === 'first_seen' ? 'using first seen variant' : 'excluded from analysis'- // Detect sample ratio mismatch (p < 0.001 is significant)- const hasSRM = hasSampleRatioMismatch(exposures)const srmLine = sampleRatioLine(exposures, healthUi === 'panel')+ const hasSRM = reportsMismatch(srmLine)// The open panel says "No exposures yet" for a draft and after a failed exposure query too.// Only a launched experiment with an answer that holds no exposure is the zero-exposure state.@@ -496,12 +500,14 @@ export function Exposures(): JSX.Element {/>{srmLine && exposures?.sample_ratio_mismatch != null && (<div className="flex items-center gap-1 text-xs mt-2">- {srmLine.tone === 'warning' ? (+ {reportsMismatch(srmLine) ? (<><Tooltip title={srmLine.tooltip}>- <span className="flex items-center gap-1 text-warning cursor-pointer">- <IconWarning className="text-sm" />- <span className="font-semibold">{srmLine.label}</span>+ <span+ className={`flex items-center gap-1 cursor-pointer ${MISMATCH_LINE_STYLE[srmLine.tone].className}`}+ >+ {MISMATCH_LINE_STYLE[srmLine.tone].icon}+ <span>{srmLine.label}</span></span></Tooltip><span className="text-muted">diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.pyindex 89831703d782..75b03bb25960 100644--- a/products/experiments/backend/health/checks/forced_variant.py+++ b/products/experiments/backend/health/checks/forced_variant.py@@ -60,6 +60,13 @@ class _Pins:graded: PinnedConditions+def assigns_nobody_by_split(flag: FlagState) -> bool:+ """Whether every release condition that includes users sets a variant override, so the split assigns nobody."""+ return bool(flag.reachable_conditions) and all(+ condition.pinned_variant is not None for condition in flag.reachable_conditions+ )++def overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:"""The overrides in place that SOME_CONDITIONS_PINNED is graded by, or None when it is not graded."""flag = flag_to_check(ctx, before_launch=True)@@ -121,7 +128,7 @@ def _pins(ctx: HealthContext, flag: FlagState) -> _Pins | None:analyzed = tuple(condition for condition in pinned if condition.pinned_variant not in ctx.excluded_variant_keys)return _Pins(pinned=pinned,- all_pinned=len(pinned) == len(flag.reachable_conditions),+ all_pinned=assigns_nobody_by_split(flag),analyzed=analyzed,graded=pinned_conditions(analyzed, flag.variants),)@@ -191,7 +198,7 @@ def _set_after_launch(ctx: HealthContext, numbers: Sequence[int]) -> _SetAfterLa"who were exposed before may have switched variants.")else:- names = _set_names(sorted(override.condition_number for override in added))+ names = set_names(sorted(override.condition_number for override in added))sentence = (f" The overrides of {names} were set after launch, last on {at}, so users they match who were exposed ""before may have switched variants."@@ -212,7 +219,7 @@ def _quoted(keys: Sequence[str]) -> str:return ", ".join(f'"{key}"' for key in keys)-def _set_names(numbers: Sequence[int]) -> str:+def set_names(numbers: Sequence[int]) -> str:if len(numbers) == 1:return f"Set {numbers[0]}"return "Sets " + ", ".join(str(number) for number in numbers[:-1]) + f" and {numbers[-1]}"@@ -275,13 +282,13 @@ def _some_pinned_detail(key = pinned_keys[0]overrides, them = "the override", "it"fact = (- f'{_set_names(numbers)} sets the variant override "{key}", so users who match it always get "{key}" '+ f'{set_names(numbers)} sets the variant override "{key}", so users who match it always get "{key}" '"instead of a random variant.")else:overrides, them = "the overrides", "them"fact = (- f"{_set_names(numbers)} set variant overrides ({_quoted(pinned_keys)}), so users who match them "+ f"{set_names(numbers)} set variant overrides ({_quoted(pinned_keys)}), so users who match them ""always get the variant of their condition instead of a random one.")why = (diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.pyindex 391a203a3299..dfc1d995b324 100644--- a/products/experiments/backend/health/checks/srm.py+++ b/products/experiments/backend/health/checks/srm.py@@ -1,3 +1,4 @@+import mathimport unicodedatafrom collections.abc import Mappingfrom datetime import date@@ -17,14 +18,30 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,)+from products.experiments.backend.health.checks.forced_variant import (+ SOME_CONDITIONS_PINNED,+ assigns_nobody_by_split,+ set_names,+)from products.experiments.backend.health.checks.split_changed import split_changed_mid_run-from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, FlagHistory, HealthContext+from products.experiments.backend.health.context import (+ ExposureSegment,+ ExposureTotals,+ FlagHistory,+ FlagState,+ HealthContext,+ ReachableCondition,+)+from products.experiments.backend.health.pinned_share import pinned_variant_keysfrom products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicablefrom products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.feature_flags.backend.facade.edits import SplitChangeSAMPLE_RATIO_MISMATCH_P_VALUE = 0.001SRM_MINIMUM_SAMPLE_SIZE = 100+# The share of persons or groups that would have to switch variants to match the split. Below it, a mismatch moves too+# few of them to change the results much, so it is info. A large experiment reaches the p-value with a far smaller skew.+MATERIAL_SHARE_OFF_SPLIT = 0.01TOP_SEGMENT_VALUES = 5# Below this, a skew of a few points passes the homogeneity test for lack of power, and would read as the same skew# everywhere. Two segments this large are needed before the finding calls a dimension uniform.@@ -91,6 +108,7 @@ def reads_segments(ctx: HealthContext) -> bool:outcome = srm(ctx)if not isinstance(outcome, ExperimentHealthFinding):return False+ # A variant override can explain a mismatch, but a segment that locates it names the cause more precisely.if outcome.subcode != SPLIT_CHANGED:return Trueretest_p_value = outcome.evidence.get("p_value_after_split_change")@@ -102,7 +120,7 @@ def segments_first_exposed_after(ctx: HealthContext) -> date | None:change, the users first exposed before it got their variants from another split, so they would fail the testagainst the current split in any segment they fill."""history = ctx.flag_history- if not isinstance(history, FlagHistory) or _split_change_warning(ctx) is None:+ if not isinstance(history, FlagHistory) or _split_change_cause(ctx) is None:return Nonereturn _last_split_change_day(history)@@ -111,6 +129,10 @@ def srm(ctx: HealthContext) -> CheckOutcome:exposures = ctx.exposuresif exposures is None:return not_applicable(RUNS_AFTER_LAUNCH)+ # The split then serves nobody, so the test compares nothing. `forced_variant_release_condition` explains it.+ overrides = _flag_of_the_run(ctx)+ if overrides is not None and assigns_nobody_by_split(overrides):+ return not_applicable("The split assigns nobody, because every release condition sets a variant override")p_value = exposures.sample_ratio_mismatch_p_valueif p_value is None:# The exposure query compares only the variants with a share of the split that the analysis keeps, so after@@ -125,16 +147,55 @@ def srm(ctx: HealthContext) -> CheckOutcome:return not_applicable("Runs once enough users are exposed")if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:return None+ share_off_split = _share_off_split(exposures.total_exposures, exposures.sample_ratio_expected)split_change = _split_change_warning(ctx)history = ctx.flag_history- if split_change is None or not isinstance(history, FlagHistory):- return _mismatch(exposures, p_value)- # The warning stays a warning: the users exposed after the change are fewer than those the test covers, so a- # retest that passes can't rule out a mismatch with another cause. The retest only adds evidence.+ ruled_out = False+ if split_change is not None and isinstance(history, FlagHistory):+ if not _skew_predates_the_change(ctx, exposures, history):+ return _after_split_change(ctx, exposures, history, split_change, p_value, share_off_split)+ ruled_out = True+ cause = _segment_cause(exposures, exposures.unit)+ pinned = _pinned_cause(ctx, exposures)+ if pinned and not cause.locates_the_mismatch:+ return _pinned_mismatch(exposures, pinned, p_value, share_off_split, cause)+ return _mismatch(exposures, p_value, share_off_split, cause, split_change_ruled_out=ruled_out)+++def _severity(share_off_split: float) -> ExperimentHealthFindingSeverity:+ if share_off_split < MATERIAL_SHARE_OFF_SPLIT:+ return ExperimentHealthFindingSeverity.INFO+ return ExperimentHealthFindingSeverity.WARNING+++def _small_mismatch(share_off_split: float, unit: str) -> list[str]:+ if share_off_split >= MATERIAL_SHARE_OFF_SPLIT:+ return []+ # Rounded down, so that a share just under the floor doesn't read as the floor. The epsilon keeps a share of 0.008+ # that float arithmetic computes as 0.00799… at 0.8%.+ tenths = math.floor(share_off_split * 1000 + 1e-9)+ share = f"Only {tenths / 10:g}%" if tenths else "Fewer than 0.1%"+ return [+ f"{share} of {unit}s would have to switch variants to match the split, so the difference is unlikely to "+ "change the results."+ ]+++def _after_split_change(+ ctx: HealthContext,+ exposures: ExposureTotals,+ history: FlagHistory,+ split_change: ExperimentHealthFinding,+ p_value: float,+ share_off_split: float,+) -> ExperimentHealthFinding:+ # A retest that passes keeps the severity: the users exposed after the change are fewer than those the test covers,+ # so it can't rule out a mismatch with another cause. The retest only adds evidence.unit = exposures.unitretest, p_value_after_change = _retest_after_change(ctx, exposures, history, unit)evidence: dict[str, str | int | float | None] = {"p_value": p_value,+ "share_off_split": round(share_off_split, 4),"p_value_after_split_change": p_value_after_change,}cause = (@@ -148,12 +209,13 @@ def srm(ctx: HealthContext) -> CheckOutcome:code=ExperimentHealthFindingCode.SRM,subcode=SPLIT_CHANGED,reason=cause.reason if cause is not None else None,- severity=ExperimentHealthFindingSeverity.WARNING,+ severity=_severity(share_off_split),title=f"{unit.capitalize()}s are not split as configured after a split change",detail=" ".join([f"The distribution of {unit}s across variants doesn't match your current rollout percentages "f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",+ *_small_mismatch(share_off_split, unit),f"The variant split changed after launch while {unit}s were exposed, and some of them got their ""variants from another split, which can cause this.",*retest,@@ -207,6 +269,49 @@ def _last_split_change_day(history: FlagHistory) -> date:return history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()+def _split_change_cause(ctx: HealthContext) -> ExperimentHealthFinding | None:+ """The warning of the split change check, when the change can explain the mismatch."""+ warning = _split_change_warning(ctx)+ history = ctx.flag_history+ if warning is None or not isinstance(history, FlagHistory) or ctx.exposures is None:+ return None+ return None if _skew_predates_the_change(ctx, ctx.exposures, history) else warning+++def _skew_predates_the_change(ctx: HealthContext, exposures: ExposureTotals, history: FlagHistory) -> bool:+ """Whether the users first exposed before the first split change, those first exposed after the last one and all+ exposed users show the same shares, and those after it don't match the current split. Mixing assignments then+ can't explain the mismatch, because each group of users alone shows it."""+ events = ctx.exposure_events+ if events is None or events.activation is not None:+ return False+ changes = history.counted(SplitChange)+ if not changes:+ return False+ first_day = changes[0].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()+ last_day = _last_split_change_day(history)+ weights = exposures.sample_ratio_expected+ # A user first exposed on the day of a change can have either split, so neither group counts them.+ before = {+ variant: sum(count for day, count in exposures.daily_exposures.get(variant, {}).items() if day < first_day)+ for variant in weights+ }+ after = {+ variant: sum(count for day, count in exposures.daily_exposures.get(variant, {}).items() if day > last_day)+ for variant in weights+ }+ retest = sample_ratio_test(after, weights)+ if retest is None or retest.p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:+ return False+ if sum(before.values()) < SRM_MINIMUM_SAMPLE_SIZE:+ return False+ # The users between the changes and on a change day fill neither group, so all users must show the shares too.+ return (+ _share_off_split(before, after) < MATERIAL_SHARE_OFF_SPLIT+ and _share_off_split(exposures.total_exposures, after) < MATERIAL_SHARE_OFF_SPLIT+ )++def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:try:outcome = split_changed_mid_run(ctx)@@ -219,23 +324,113 @@ def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:return None-def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:+def _flag_of_the_run(ctx: HealthContext) -> FlagState | None:+ """The flag whose variant overrides can explain the mismatch. None after the end, because the flag then often+ holds the shipped decision and not the setup of the run, like `flag_to_check` says."""+ return None if ctx.has_ended else ctx.flag+++def _pinned_cause(ctx: HealthContext, exposures: ExposureTotals) -> tuple[ReachableCondition, ...]:+ """The release conditions with a variant override in the test, when one of them pins the variant furthest above+ its expected count. Their users don't follow the split, so they can cause the mismatch."""+ expected = exposures.sample_ratio_expected+ flag = _flag_of_the_run(ctx)+ if flag is None or not expected:+ return ()+ pinned = tuple(condition for condition in flag.reachable_conditions if condition.pinned_variant in expected)+ over = max(expected, key=lambda variant: exposures.total_exposures.get(variant, 0) - expected[variant])+ return pinned if any(condition.pinned_variant == over for condition in pinned) else ()+++def _pinned_mismatch(+ exposures: ExposureTotals,+ pinned: tuple[ReachableCondition, ...],+ p_value: float,+ share_off_split: float,+ segments: "_SegmentCause",+) -> ExperimentHealthFinding:unit = exposures.unit- cause = _segment_cause(exposures, unit)+ numbers = [condition.number for condition in pinned]+ keys = pinned_variant_keys(pinned)+ if len(numbers) == 1:+ cause = (+ f'{set_names(numbers)} sets the variant override "{keys[0]}", so the {unit}s it matches get "{keys[0]}" '+ "instead of a variant from the split, which can cause this."+ )+ overrides = "the override"+ else:+ quoted = ", ".join(f'"{key}"' for key in keys)+ cause = (+ f"{set_names(numbers)} set variant overrides ({quoted}), so the {unit}s they match get the variant of "+ "their condition instead of one from the split, which can cause this."+ )+ overrides = "the overrides"+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.SRM,+ subcode=SOME_CONDITIONS_PINNED,+ reason=None,+ severity=_severity(share_off_split),+ title=f"{unit.capitalize()}s are not split as configured because of variant overrides",+ detail=" ".join(+ [+ f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",+ *_small_mismatch(share_off_split, unit),+ cause,+ "If these are internal or test users, check that the filter for internal and test users excludes "+ f"them from the results. Otherwise remove {overrides}.",+ ]+ ),+ evidence={+ "p_value": p_value,+ "share_off_split": round(share_off_split, 4),+ "pinned_conditions": ",".join(str(number) for number in numbers),+ "pinned_variants": ",".join(keys),+ **segments.evidence,+ },+ actions=(),+ diagnostic_ref="A2",+ )+++def _mismatch(+ exposures: ExposureTotals,+ p_value: float,+ share_off_split: float,+ cause: "_SegmentCause",+ *,+ split_change_ruled_out: bool,+) -> ExperimentHealthFinding:+ unit = exposures.unit+ ruled_out = (+ [+ f"The variant split changed after launch, but the {unit}s first exposed before the change show the same "+ "mismatch as those first exposed after it, so the change doesn't explain it."+ ]+ if split_change_ruled_out+ else []+ )return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=None,reason=cause.reason,- severity=ExperimentHealthFindingSeverity.WARNING,+ severity=_severity(share_off_split),title=f"{unit.capitalize()}s are not split across variants as configured",detail=" ".join([f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",+ *_small_mismatch(share_off_split, unit),+ *ruled_out,*(cause.sentences or ("This may indicate issues with randomization or data collection.",)),]),- evidence={"p_value": p_value, **cause.evidence},+ evidence={+ "p_value": p_value,+ "share_off_split": round(share_off_split, 4),+ **({"split_change_ruled_out": True} if split_change_ruled_out else {}),+ **cause.evidence,+ },actions=(),diagnostic_ref="A2",)@@ -266,6 +461,10 @@ class _SegmentCause:reason: str | Noneevidence: dict[str, str | int | float | None]+ @property+ def locates_the_mismatch(self) -> bool:+ return self.reason is not None and self.reason != SrmReason.UNIFORM+def _segment_cause(exposures: ExposureTotals, unit: str) -> _SegmentCause:tests = _segment_tests(exposures)@@ -314,17 +513,19 @@ def _verdict(tests: list[_SegmentTest], dimension: ExposureSegmentDimension, weihomogeneity_p_value=homogeneity_p_value,powered=powered,located=located,- distance=_distance_from_split(located, weights) if located is not None else 0.0,+ distance=_share_off_split(located.exposed_by_variant, weights) if located is not None else 0.0,)-def _distance_from_split(segment: ExposureSegment, weights: Mapping[str, float]) -> float:- """The share of the segment's persons or groups that would have to move between variants to match the split."""- total = sum(segment.exposed_by_variant.get(variant, 0) for variant in weights)+def _share_off_split(exposed_by_variant: Mapping[str, int], weights: Mapping[str, float]) -> float:+ """The share of the persons or groups that would have to move between variants to match the weights."""+ total = sum(exposed_by_variant.get(variant, 0) for variant in weights)total_weight = sum(weights.values())+ if total <= 0 or total_weight <= 0:+ return 0.0return (sum(- abs(segment.exposed_by_variant.get(variant, 0) / total - weight / total_weight)+ abs(exposed_by_variant.get(variant, 0) / total - weight / total_weight)for variant, weight in weights.items())/ 2diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex b2d1e15445fe..d668a72166fc 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -134,7 +134,8 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (# The exposure query runs these on its answer. They must not include the checks above: the exposure answer# is cached for a day, and a flag edit would not clear their findings. `srm` reads the verdict of-# `split_changed_mid_run`, which only a split edit can change, and the split is part of the exposure query's cache key.+# `split_changed_mid_run`, which only a split edit can change, and the variant overrides of the release conditions. The+# split and the release conditions are part of the exposure query's cache key.EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (HealthCheck(key=ExperimentHealthCheckKey.ZERO_EXPOSURES,diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex edf68d62901f..6c52472fb08f 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -664,6 +664,63 @@ class TestSrmAfterSplitChange(TestCase):self.assertEqual((reads_flag_history(ctx), reads_segments(ctx), segments_first_exposed_after(ctx)), expected)+ @parameterized.expand(+ [+ (+ "same_skew_before_the_change",+ {"control": 600, "test": 400},+ {"control": 0, "test": 0},+ (None, True, None),+ "The variant split changed after launch, but the users first exposed before the change show the same "+ "mismatch as those first exposed after it, so the change doesn't explain it.",+ ),+ (+ "another_skew_before_the_change",+ {"control": 500, "test": 500},+ {"control": 0, "test": 0},+ ("split_changed", True, date(2026, 9, 14)),+ "don't match the current split either (p < 0.001), so look for another cause too.",+ ),+ (+ "another_skew_on_the_change_day",+ {"control": 600, "test": 400},+ {"control": 5000, "test": 0},+ ("split_changed", True, date(2026, 9, 14)),+ "don't match the current split either (p < 0.001), so look for another cause too.",+ ),+ ]+ )+ def test_rules_out_a_split_change_when_the_skew_predates_it(+ self,+ _name: str,+ before_change: dict[str, int],+ on_change_day: dict[str, int],+ expected: tuple[str | None, bool, date | None],+ text: str,+ ) -> None:+ after_change = {"control": 600, "test": 400}+ days = {date(2026, 9, 12): before_change, date(2026, 9, 14): on_change_day, date(2026, 9, 15): after_change}+ total_exposures = {variant: sum(day[variant] for day in days.values()) for variant in after_change}+ whole_run = sum(total_exposures.values())+ totals = replace(+ _totals(after_change),+ total_exposures=total_exposures,+ sample_ratio_expected={"control": 0.8 * whole_run, "test": 0.2 * whole_run},+ daily_exposures={+ variant: {day: day_counts[variant] for day, day_counts in days.items()} for variant in after_change+ },+ )+ ctx = replace(+ _context(EVEN, UNEVEN, exposed_units=5, first_change_at=datetime(2026, 9, 14, 13, 4, tzinfo=UTC)),+ exposures=totals,+ )++ outcome = srm(ctx)++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual((outcome.subcode, reads_segments(ctx), segments_first_exposed_after(ctx)), expected)+ self.assertIn(text, outcome.detail)+def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.pyindex 06687664b0f7..861bd9602317 100644--- a/products/experiments/backend/health/test/test_srm.py+++ b/products/experiments/backend/health/test/test_srm.py@@ -8,9 +8,10 @@ from posthog.schema import MultipleVariantHandlingfrom posthog.hogql_queries.utils.breakdowns import BREAKDOWN_NULL_STRING_LABEL, BREAKDOWN_OTHER_STRING_LABEL-from products.experiments.backend.facade.contracts import ExperimentHealthFinding-from products.experiments.backend.health.checks.srm import srm-from products.experiments.backend.health.context import ExposureSegment, ExposureTotals+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverity+from products.experiments.backend.health.checks.srm import reads_segments, srm+from products.experiments.backend.health.context import ExposureSegment, ExposureTotals, parse_flag_state+from products.experiments.backend.health.skips import CheckSkipfrom products.experiments.backend.health.test.factories import RUNNING_CONTEXTfrom products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension@@ -211,6 +212,7 @@ class TestSrmSegments(TestCase):evidence,{"p_value": 0.00001,+ "share_off_split": 0.1,"located_dimensions": "library","library_1": "web","library_1_p_value": 1.0,@@ -230,3 +232,169 @@ class TestSrmSegments(TestCase):"page_2_exposed_test": 200,},)+++EVEN_SPLIT = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]+EVERYONE = {"properties": [], "rollout_percentage": 100}+++PINNED_TEST = (+ f'{MISMATCH} Set 1 sets the variant override "test", so the users it matches get "test" instead of a variant from '+ "the split, which can cause this. If these are internal or test users, check that the filter for internal and test "+ "users excludes them from the results. Otherwise remove the override."+)+++def _pinned_to(variant: str) -> dict[str, object]:+ return {+ "properties": [{"key": "$geoip_country_code", "value": ["DE"], "operator": "exact", "type": "person"}],+ "rollout_percentage": 100,+ "variant": variant,+ }+++class TestSrmCauses(TestCase):+ @parameterized.expand(+ [+ (+ "half_a_percent_off",+ {"control": 50500, "test": 49500},+ {"control": 50000, "test": 50000},+ ExperimentHealthFindingSeverity.INFO,+ f"{MISMATCH} Only 0.5% of users would have to switch variants to match the split, so the difference "+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",+ ),+ (+ "three_variants_under_one_percent_off",+ {"control": 33200, "test": 33300, "blue": 33500},+ {"control": 34000, "test": 33000, "blue": 33000},+ ExperimentHealthFindingSeverity.INFO,+ f"{MISMATCH} Only 0.8% of users would have to switch variants to match the split, so the difference "+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",+ ),+ (+ "just_under_one_percent_off",+ {"control": 50990, "test": 49010},+ {"control": 50000, "test": 50000},+ ExperimentHealthFindingSeverity.INFO,+ f"{MISMATCH} Only 0.9% of users would have to switch variants to match the split, so the difference "+ "is unlikely to change the results. This may indicate issues with randomization or data collection.",+ ),+ (+ "under_a_tenth_of_a_percent_off",+ {"control": 5002000, "test": 4998000},+ {"control": 5000000, "test": 5000000},+ ExperimentHealthFindingSeverity.INFO,+ f"{MISMATCH} Fewer than 0.1% of users would have to switch variants to match the split, so the "+ "difference is unlikely to change the results. This may indicate issues with randomization or data "+ "collection.",+ ),+ (+ "one_percent_off",+ {"control": 51000, "test": 49000},+ {"control": 50000, "test": 50000},+ ExperimentHealthFindingSeverity.WARNING,+ NO_LOCATION,+ ),+ ]+ )+ def test_a_mismatch_of_under_one_percent_of_users_is_info(+ self,+ _name: str,+ total_exposures: dict[str, int],+ expected: dict[str, float],+ severity: ExperimentHealthFindingSeverity,+ detail: str,+ ) -> None:+ exposures = replace(_exposures(None), total_exposures=total_exposures, sample_ratio_expected=expected)++ outcome = srm(replace(RUNNING_CONTEXT, exposures=exposures))++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual((outcome.severity, outcome.detail), (severity, detail))++ @parameterized.expand(+ [+ (+ "override_of_the_variant_in_excess",+ [_pinned_to("test"), EVERYONE],+ False,+ None,+ ("some_conditions_pinned", True),+ PINNED_TEST,+ ),+ (+ "override_and_a_uniform_mismatch",+ [_pinned_to("test"), EVERYONE],+ False,+ (_segment(LIBRARY, "web", 400, 600), _segment(LIBRARY, "posthog-node", 400, 600)),+ ("some_conditions_pinned", True),+ PINNED_TEST,+ ),+ (+ "override_and_a_segment_with_the_mismatch",+ [_pinned_to("test"), EVERYONE],+ False,+ SERVER_SKEWED,+ (None, True),+ f'{MISMATCH} The mismatch comes from users whose first exposure was sent by the "posthog-node" '+ f"library. {NEXT_STEP}",+ ),+ (+ "override_of_the_other_variant",+ [_pinned_to("control"), EVERYONE],+ False,+ None,+ (None, True),+ NO_LOCATION,+ ),+ ("override_after_the_end", [_pinned_to("test"), EVERYONE], True, None, (None, True), NO_LOCATION),+ (+ "every_condition_overridden_after_the_end",+ [_pinned_to("test"), {**EVERYONE, "variant": "control"}],+ True,+ None,+ (None, True),+ NO_LOCATION,+ ),+ ]+ )+ def test_names_a_variant_override_of_the_variant_in_excess(+ self,+ _name: str,+ groups: list[dict[str, object]],+ has_ended: bool,+ segments: tuple[ExposureSegment, ...] | None,+ expected: tuple[str | None, bool],+ detail: str,+ ) -> None:+ ctx = replace(+ RUNNING_CONTEXT,+ has_ended=has_ended,+ flag=parse_flag_state(active=True, deleted=False, groups=groups, variants=EVEN_SPLIT),+ exposures=_exposures(segments),+ )++ outcome = srm(ctx)++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(((outcome.subcode, reads_segments(ctx)), outcome.detail), (expected, detail))++ def test_leaves_a_split_that_assigns_nobody_to_the_variant_override_finding(self) -> None:+ ctx = replace(+ RUNNING_CONTEXT,+ flag=parse_flag_state(+ active=True,+ deleted=False,+ groups=[_pinned_to("test"), {**EVERYONE, "variant": "control"}],+ variants=EVEN_SPLIT,+ ),+ exposures=_exposures(None),+ )++ outcome = srm(ctx)++ assert isinstance(outcome, CheckSkip)+ self.assertEqual(+ outcome.reason, "The split assigns nobody, because every release condition sets a variant override"+ )diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex 1d23db7c25a1..4a30ba41d3c7 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -1746,6 +1746,7 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):self.assertEqual({key: value for key, value in finding.evidence.items() if not key.endswith("p_value")},{+ "share_off_split": 0.1667,"located_dimensions": "library,library_version","library_1": "web","library_1_exposed_control": 100,@@ -1934,7 +1935,7 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):ExperimentExposureHealthFindingCode.SRM,"The distribution of users across variants doesn't match your configured rollout percentages ""(p < 0.001). This may indicate issues with randomization or data collection.",- {"p_value"},+ {"p_value", "share_off_split"},)],)diff --git a/products/experiments/frontend/health/exposureHealth.test.ts b/products/experiments/frontend/health/exposureHealth.test.tsindex 3c1c49fb7b87..015d9339e3c1 100644--- a/products/experiments/frontend/health/exposureHealth.test.ts+++ b/products/experiments/frontend/health/exposureHealth.test.ts@@ -1,14 +1,19 @@-import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'+import {+ type ExperimentExposureHealthCheck,+ type ExperimentExposureQueryResponse,+ NodeKind,+} from '~/queries/schema/schema-general'import { SAMPLE_RATIO_MISMATCH_DESCRIPTION, sampleRatioLine } from './exposureHealth'const srmFinding = (- subcode: string | null+ subcode: string | null,+ severity: 'warning' | 'info' = 'warning'): NonNullable<ExperimentExposureQueryResponse['health_findings']>[number] => ({code: 'srm',subcode,reason: null,- severity: 'warning',+ severity,title: 'Users are not split as configured',detail: `Server detail for ${subcode ?? 'a mismatch'}`,evidence: {},@@ -26,6 +31,13 @@ const MISMATCH = {health_checks: [],} satisfies ExperimentExposureQueryResponse+const SRM_NOT_APPLICABLE: ExperimentExposureHealthCheck = {+ key: 'srm',+ name: 'Sample ratio mismatch',+ status: 'not_applicable',+ reason: 'The split assigns nobody, because every release condition sets a variant override',+}+describe('sampleRatioLine', () => {it.each([{@@ -48,6 +60,18 @@ describe('sampleRatioLine', () => {tooltip: 'Server detail for a mismatch',},},+ {+ desc: 'a mismatch the server reports as info',+ exposures: { ...MISMATCH, health_findings: [srmFinding(null, 'info')] },+ followsServer: true,+ expected: { tone: 'info', label: 'Sample ratio mismatch detected' },+ },+ {+ desc: 'a split that every variant override bypasses',+ exposures: { ...MISMATCH, health_findings: [], health_checks: [SRM_NOT_APPLICABLE] },+ followsServer: true,+ expected: null,+ },{desc: 'a reader without the health panel',exposures: MISMATCH,@@ -69,6 +93,6 @@ describe('sampleRatioLine', () => {},},])('follows the server only with the health panel: $desc', ({ exposures, followsServer, expected }) => {- expect(sampleRatioLine(exposures, followsServer)).toMatchObject(expected)+ expect(sampleRatioLine(exposures, followsServer)).toEqual(expected && expect.objectContaining(expected))})})diff --git a/products/experiments/frontend/health/exposureHealth.ts b/products/experiments/frontend/health/exposureHealth.tsindex 4de5e21f764a..3b96aa23efee 100644--- a/products/experiments/frontend/health/exposureHealth.ts+++ b/products/experiments/frontend/health/exposureHealth.ts@@ -24,11 +24,16 @@ export function serverSampleRatioFinding(/** What the exposures card says about the sample ratio test. */export type SampleRatioLine = {- tone: 'warning' | 'success'+ tone: 'warning' | 'info' | 'success'label: stringtooltip: string}+/** Whether the exposures card's line reports a mismatch, as a warning or as info. */+export function reportsMismatch(line: SampleRatioLine | null): line is SampleRatioLine & { tone: 'warning' | 'info' } {+ return line !== null && line.tone !== 'success'+}+/*** The exposures card's line about the sample ratio test, or null without a test. With the health panel the card uses* the server's finding, which can name a split change as the likely cause. Without it the card keeps the page's own@@ -41,10 +46,17 @@ export function sampleRatioLine(if (exposures?.sample_ratio_mismatch == null) {return null}+ // SRM doesn't apply when every release condition sets a variant override. The override finding explains it.+ if (+ followsServer &&+ exposures.health_checks?.some(({ key, status }) => key === 'srm' && status === 'not_applicable')+ ) {+ return null+ }const finding = followsServer ? serverSampleRatioFinding(exposures) : undefinedif (finding) {return {- tone: 'warning',+ tone: finding.severity === 'info' ? 'info' : 'warning',label:finding.subcode === 'split_changed'? 'Sample ratio mismatch after a split change'