:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): name the split change behind a sample ratio mismatch (#122590)
Marcel Poelker767be38c8ec1
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 6f30b2d6c2f2..a44c83985ac0 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -11085,9 +11085,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.df03873a81d660b11a8fbabb7187ae6a8ac51b16fa9e984530b7387478ebdb04.3i56n6We4-RK73B4ma2ipXHZH3gP0KdCGmWzCdq-S5E+ hash: v1.k794b7964.c42640b2d757ecf6208b3658848ee0ac7f212b167ce614aaa41f36e0f935736b.opgqM_7IOcqH8n65gLUXLw9kkYbvtRJbVjmi7qukMG8scenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.4f995cb939700d5c7474bf9fa87e2685ef5d2b481f82ba25bb3b23d8d9917129.qCBrumZsUJSGWp9aXR3Qqau-QRbmFgUJ7DbV1ELWrnI+ hash: v1.k794b7964.1249fadfb8bf915c349e9adb2500e6ee98550a973f6d044fab1b0ad1610e7091.GTAsiqSMl9rSqNoD3BBeuNvdhNMlzCSfXnT3FbHwd28scenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiwscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsxindex 93b3738d53eb..403013eb2fa8 100644--- a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx@@ -18,10 +18,12 @@ import { teamLogic } from 'scenes/teamLogic'import { ExperimentExposureCriteria, ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'import { EXPERIMENT_VARIANT_MULTIPLE } from 'products/experiments/frontend/constants'+import { healthFindingReason } from 'products/experiments/frontend/health/experimentHealthFindingEvents'import {- SAMPLE_RATIO_MISMATCH_DESCRIPTION,getTotalExposures,hasSampleRatioMismatch,+ sampleRatioLine,+ serverSampleRatioFinding,} from 'products/experiments/frontend/health/exposureHealth'import { useHealthFindingReporting } from 'products/experiments/frontend/health/useHealthFindingReporting'@@ -135,6 +137,7 @@ export function Exposures(): JSX.Element {experiment,excludedVariants,resolvedExposureEvent,+ healthUi,} = useValues(experimentLogic)const { openExposureCriteriaModal } = useActions(exposureCriteriaModalLogic)const { refreshExperimentResults } = useActions(experimentLogic)@@ -167,14 +170,28 @@ export function Exposures(): JSX.Element {// Detect sample ratio mismatch (p < 0.001 is significant)const hasSRM = hasSampleRatioMismatch(exposures)+ const srmLine = sampleRatioLine(exposures, healthUi === 'panel')// The open panel says "No exposures yet" for a draft and after a failed exposure query too.// Only a launched experiment with an answer that holds no exposure is the zero-exposure state.// The answer holds a series for every configured variant, with zero counts, so only the total tells.const hasZeroExposures = !isExperimentDraft && !exposuresLoading && exposures != null && totalExposures === 0+ // With the health panel, its row reports the finding as shown. The card reports the same finding, so that opening+ // the card's evidence counts against the panel's "shown" event and not a second one.+ const srmFinding = healthUi === 'panel' ? serverSampleRatioFinding(exposures) : undefinedconst { reportOpened: reportSrmOpened } = useHealthFindingReporting(- hasSRM && !isExperimentDraft && !exposuresLoading ? { code: 'srm' } : null+ hasSRM && !isExperimentDraft && !exposuresLoading+ ? srmFinding+ ? {+ code: 'srm',+ variant: srmFinding.subcode ?? undefined,+ severity: srmFinding.severity,+ reason: healthFindingReason(srmFinding.evidence),+ }+ : { code: 'srm' }+ : null,+ !srmFinding)// The collapsed header shows the total only, so the zero-exposure state is on screen in the open panel alone.const { reportOpened: reportZeroExposuresOpened, reportActedOn: reportZeroExposuresActedOn } =@@ -243,8 +260,8 @@ export function Exposures(): JSX.Element {))}</div>)}- {hasSRM && (- <Tooltip title={SAMPLE_RATIO_MISMATCH_DESCRIPTION}>+ {srmLine?.tone === 'warning' && (+ <Tooltip title={srmLine.tooltip}><IconWarning className="text-warning text-lg" /></Tooltip>)}@@ -478,16 +495,14 @@ export function Exposures(): JSX.Element {},]}/>- {exposures?.sample_ratio_mismatch != null && (+ {srmLine && exposures?.sample_ratio_mismatch != null && (<div className="flex items-center gap-1 text-xs mt-2">- {hasSRM ? (+ {srmLine.tone === 'warning' ? (<>- <Tooltip title={SAMPLE_RATIO_MISMATCH_DESCRIPTION}>+ <Tooltip title={srmLine.tooltip}><span className="flex items-center gap-1 text-warning cursor-pointer"><IconWarning className="text-sm" />- <span className="font-semibold">- Sample ratio mismatch detected- </span>+ <span className="font-semibold">{srmLine.label}</span></span></Tooltip><span className="text-muted">@@ -496,12 +511,10 @@ export function Exposures(): JSX.Element {</>) : (<>- <Tooltip title="No sample ratio mismatch detected. The difference between actual and expected exposures is within normal random variation.">+ <Tooltip title={srmLine.tooltip}><span className="flex items-center gap-1 text-success cursor-pointer"><IconCheckCircle className="text-sm" />- <span>- Exposure distribution matches rollout percentages- </span>+ <span>{srmLine.label}</span></span></Tooltip><span className="text-muted">diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.tsindex f4db3da14100..a1817f6d8d44 100644--- a/frontend/src/scenes/experiments/experimentLogic.test.ts+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts@@ -131,10 +131,11 @@ describe('experimentLogic', () => {total_exposures: { control: 600, test: 350, $multiple: 50 },sample_ratio_mismatch: { expected: { control: 475, test: 475 }, p_value: 0.0001 },bias_risk: { multiple_variant_percentage: 5 },- health_findings: [{ code: 'srm' }, { code: 'bias_risk_multiple_excluded' }],+ health_findings: [{ code: 'srm', subcode: 'split_changed' }, { code: 'bias_risk_multiple_excluded' }],}const noServerVerdicts = {srm_server: null,+ srm_cause: null,zero_exposures_server: null,zero_exposures_cause: null,bias_risk_server: null,@@ -176,6 +177,7 @@ describe('experimentLogic', () => {has_srm: true,has_bias_risk: true,srm_server: true,+ srm_cause: 'split_changed',zero_exposures_server: false,bias_risk_server: true,},diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.pyindex 6291aa38a068..c37a2ec2a6a8 100644--- a/products/experiments/backend/health/checks/flag_change_findings.py+++ b/products/experiments/backend/health/checks/flag_change_findings.py@@ -81,8 +81,8 @@ def history_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked# `flag_variants_mismatch` reports a flag without variants, and the variants have to come back first.if not any(variant.key for variant in flag.variants):return not_applicable(NEEDS_VARIANTS)- # Only the health endpoint loads the history, and it does for every state that reaches here, so None there- # means that the load failed.+ # The health endpoint loads the history for every state that reaches here, and so does the exposure answer before+ # `srm` reads it, so None means that the load failed.if ctx.flag_history is None:return not_checked("Couldn't check because the flag history failed to load")return ctx.flag_historydiff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.pyindex 91646d197203..54e3e8b0d379 100644--- a/products/experiments/backend/health/checks/srm.py+++ b/products/experiments/backend/health/checks/srm.py@@ -1,18 +1,59 @@+from collections.abc import Mapping+from zoneinfo import ZoneInfo++from scipy.stats import chisquare++from posthog.schema import SampleRatioMismatch++from posthog.exceptions_capture import capture_exception+from products.experiments.backend.facade.contracts import (ExperimentHealthFinding,ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,)-from products.experiments.backend.health.context import HealthContext+from products.experiments.backend.health.checks.split_changed import split_changed_mid_run+from products.experiments.backend.health.context import ExposureTotals, FlagHistory, HealthContextfrom products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable+from products.feature_flags.backend.facade.edits import SplitChangeSAMPLE_RATIO_MISMATCH_P_VALUE = 0.001+SRM_MINIMUM_SAMPLE_SIZE = 100++# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.+SPLIT_CHANGED = "split_changed"+++def sample_ratio_test(observed: Mapping[str, int], weights: Mapping[str, float]) -> SampleRatioMismatch | None:+ """Chi-squared goodness-of-fit test of the observed counts against the weights of the variants in the split.+ None when there is too little data."""+ # Iterate over the weights, not the observed counts, so that a variant with 0 exposures still enters the+ # chi-square and sum(observed) == sum(expected).+ total_observed = sum(observed.get(key, 0) for key in weights)+ if total_observed < SRM_MINIMUM_SAMPLE_SIZE:+ return None+ # After dropping holdout and excluded variants, the remaining percentages may not sum to 100.+ # Normalize so that the chi-square expected counts sum to total_observed.+ total_weight = sum(weights.values())+ if total_weight <= 0 or len(weights) < 2:+ return None+ expected = {key: (weight / total_weight) * total_observed for key, weight in weights.items()}+ _, p_value = chisquare([float(observed.get(key, 0)) for key in weights], list(expected.values()))+ return SampleRatioMismatch(expected=expected, p_value=float(p_value))+++def reads_flag_history(ctx: HealthContext) -> bool:+ """Whether `srm` reads the flag history in this context. The exposure query loads the history only then, because+ the load reads the flag's activity log and can query the exposures."""+ p_value = ctx.exposures.sample_ratio_mismatch_p_value if ctx.exposures else None+ return p_value is not None and p_value < SAMPLE_RATIO_MISMATCH_P_VALUEdef srm(ctx: HealthContext) -> CheckOutcome:- if ctx.exposures is None:+ exposures = ctx.exposures+ if exposures is None:return not_applicable(RUNS_AFTER_LAUNCH)- p_value = ctx.exposures.sample_ratio_mismatch_p_value+ p_value = exposures.sample_ratio_mismatch_p_valueif p_value is None:# The exposure query compares only the variants with a share of the split that the analysis keeps, so after# a variant ships there is nothing to compare however many users are exposed.@@ -26,13 +67,92 @@ def srm(ctx: HealthContext) -> CheckOutcome:return not_applicable("Runs once enough users are exposed")if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:return None+ split_change = _split_change_warning(ctx)+ history = ctx.flag_history+ if split_change is None or not isinstance(history, FlagHistory):+ return _mismatch(exposures, p_value)+ # The warning stays a warning: the users exposed after the change are fewer than those the test covers, so a+ # retest that passes can't rule out a mismatch with another cause. The retest only adds evidence.+ unit = "group" if exposures.counts_groups else "user"+ retest, p_value_after_change = _retest_after_change(ctx, exposures, history, unit)+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.SRM,+ subcode=SPLIT_CHANGED,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title=f"{unit.capitalize()}s are not split as configured after a split change",+ detail=" ".join(+ [+ f"The distribution of {unit}s across variants doesn't match your current rollout percentages "+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}).",+ f"The variant split changed after launch while {unit}s were exposed, and some of them got their "+ "variants from another split, which can cause this.",+ *retest,+ f'Follow the steps in "{split_change.title}" to fix the results.',+ ]+ ),+ evidence={"p_value": p_value, "p_value_after_split_change": p_value_after_change},+ actions=(),+ diagnostic_ref="A2",+ )+++def _retest_after_change(+ ctx: HealthContext, exposures: ExposureTotals, history: FlagHistory, unit: str+) -> tuple[list[str], float | None]:+ """The sentence about the test of the users first exposed after the day of the last split change, who got their+ variants from the current split alone, and its p-value."""+ events = ctx.exposure_events+ if events is None:+ return [], None+ # With an activation, the exposure query dates a user by the activation, but the flag call before it assigned the+ # variant. Users dated after the change can then carry a variant from before it.+ if events.activation is not None:+ return [+ f"The exposure criteria count {unit}s at an activation, so the {unit}s exposed after the change can't "+ "be tested alone."+ ], None+ changed_on = history.counted(SplitChange)[-1].changed_at.astimezone(ZoneInfo(history.team_timezone)).date()+ after_change = sample_ratio_test(+ {+ variant: sum(count for day, count in daily.items() if day > changed_on)+ for variant, daily in exposures.daily_exposures.items()+ },+ exposures.sample_ratio_expected,+ )+ if after_change is None:+ return [f"Too few {unit}s were first exposed after the day of the change to test them alone."], None+ if after_change.p_value < SAMPLE_RATIO_MISMATCH_P_VALUE:+ return [+ f"The {unit}s first exposed after the day of the change don't match the current split either "+ f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}), so look for another cause too."+ ], after_change.p_value+ return [+ f"The {unit}s first exposed after the day of the change match the current split "+ f"(p = {after_change.p_value:.2g})."+ ], after_change.p_value+++def _split_change_warning(ctx: HealthContext) -> ExperimentHealthFinding | None:+ try:+ outcome = split_changed_mid_run(ctx)+ except Exception as error:+ # The health endpoint reports the failure of the split check itself. Here it must not hide the mismatch.+ capture_exception(error)+ return None+ if isinstance(outcome, ExperimentHealthFinding) and outcome.severity == ExperimentHealthFindingSeverity.WARNING:+ return outcome+ return None+++def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFinding:+ unit = "group" if exposures.counts_groups else "user"return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=None,severity=ExperimentHealthFindingSeverity.WARNING,- title="Users are not split across variants as configured",+ title=f"{unit.capitalize()}s are not split across variants as configured",detail=(- "The distribution of users across variants doesn't match your configured rollout percentages "+ f"The distribution of {unit}s across variants doesn't match your configured rollout percentages "f"(p < {SAMPLE_RATIO_MISMATCH_P_VALUE}). This may indicate issues with randomization or data collection."),evidence={"p_value": p_value},diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex 67b41721f910..a9d058734785 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -1,5 +1,5 @@from collections.abc import Mapping-from datetime import datetime, timedelta+from datetime import date, datetime, timedeltafrom enum import StrEnumfrom typing import Any@@ -220,6 +220,11 @@ class ExposureTotals:multiple_variant_handling: MultipleVariantHandling# None when the exposure query had too little data for the sample ratio test.sample_ratio_mismatch_p_value: float | None+ # The expected count of each variant that the sample ratio test compares, in the shares of the current split, and+ # the persons or groups first exposed on each day of the team's time zone, by variant. Only the exposure query+ # fills them, for the retest of `srm`, so both are empty elsewhere and without the test.+ sample_ratio_expected: Mapping[str, float]+ daily_exposures: Mapping[str, Mapping[date, int]]hours_since_launch: float | None# Whether the totals count groups rather than persons, by the rule the exposure query uses.counts_groups: bool@@ -294,7 +299,8 @@ class HealthContext:# ingestion never writes an event definition for is absent, because its absence proves nothing.events_received: Mapping[str, bool]exposures: ExposureTotals | None- # None when not loaded. Only the health endpoint loads it, because it can query the exposures.+ # None when not loaded. The health endpoint loads it. The exposure answer loads it only when the sample ratio test+ # fails, because the load can query the exposures and `srm` is the only exposure check that reads it.flag_history: FlagHistory | FlagHistoryNotChecked | None# None when not loaded or unknown. Only the health endpoint loads it, because it queries the events.condition_set_traffic: ConditionSetTraffic | Nonediff --git a/products/experiments/backend/health/exposure_gate.py b/products/experiments/backend/health/exposure_gate.pyindex 12073243fdba..abb39c9beb43 100644--- a/products/experiments/backend/health/exposure_gate.py+++ b/products/experiments/backend/health/exposure_gate.py@@ -147,6 +147,20 @@ def load_flag_history(return None+def load_split_history(+ experiment: Experiment, *, reader_access: ExperimentHealthReaderAccess+) -> FlagHistory | FlagHistoryNotChecked | None:+ """+ The flag history as `load_flag_history` returns it, with the exposure questions of the split changes only. The+ questions that only the other kinds read are absent. Never raises.+ """+ try:+ return _load_flag_history(experiment, reader_access, asked=frozenset({ExperimentHealthFlagChangeKind.SPLIT}))+ except Exception as error:+ capture_exception(error)+ return None++def cache_failed_scan(cache_key: str, error: Exception) -> None:"""Caches the answer of a failed scan as UNKNOWN, and reports an error that is not expected."""# The page calls the endpoint on every load, so a scan that fails must not run again on every load.@@ -443,7 +457,10 @@ class _ConditionAggregationMismatch(Exception):def _load_flag_history(- experiment: Experiment, reader_access: ExperimentHealthReaderAccess+ experiment: Experiment,+ reader_access: ExperimentHealthReaderAccess,+ *,+ asked: frozenset[ExperimentHealthFlagChangeKind] = frozenset(_KINDS),) -> FlagHistory | FlagHistoryNotChecked | None:launched_at = experiment.start_date# django-stubs types the id as int, but an unsaved instance can carry None.@@ -508,7 +525,7 @@ def _load_flag_history(exposed_units_before_last={},)times = _change_times(changes)- if not times:+ if not any(changes_of_kind(changes, _KINDS[kind]) for kind in asked):return historyreason = _not_checked_reason(experiment, reader_access)if reason is not None:@@ -528,7 +545,7 @@ def _load_flag_history(units_windows = {kind: timeline.units_window(start=launched_at, end=of_kind[-1].changed_at)for kind, change_type in _KINDS.items()- if (of_kind := changes_of_kind(changes, change_type))+ if kind in asked and (of_kind := changes_of_kind(changes, change_type))}gate = _ExposureGate(experiment,@@ -548,13 +565,17 @@ def _load_flag_history(exposed_units_before_last = {kind: gate.exposed_units(units_windows[kind])for kind, change_type in _KINDS.items()- if history.counted(change_type)+ if kind in asked and history.counted(change_type)}return replace(history,exposed_units_before_last=exposed_units_before_last,- exposed_between=_exposed_between_set_backs(gate, timeline, history),- exposed_during_overrides=_exposed_during_overrides(gate, timeline, history),+ exposed_between=_exposed_between_set_backs(gate, timeline, history, asked),+ exposed_during_overrides=(+ _exposed_during_overrides(gate, timeline, history)+ if ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS in asked+ else {}+ ),)@@ -653,23 +674,34 @@ def _exposed_between(gate: _ExposureGate, timeline: _Timeline, *, start: datetimdef _exposed_between_set_backs(- gate: _ExposureGate, timeline: _Timeline, history: FlagHistory+ gate: _ExposureGate,+ timeline: _Timeline,+ history: FlagHistory,+ asked: frozenset[ExperimentHealthFlagChangeKind],) -> dict[ExperimentHealthFlagChangeKind, bool | None]:"""Asks, per kind, whether anyone was exposed between the first and the last counted change, when the last setsback what the first changed."""exposed_between: dict[ExperimentHealthFlagChangeKind, bool | None] = {}split = history.counted(SplitChange)- if split and same_assignment(split[0].before, split[-1].after):+ if ExperimentHealthFlagChangeKind.SPLIT in asked and split and same_assignment(split[0].before, split[-1].after):exposed_between[ExperimentHealthFlagChangeKind.SPLIT] = _exposed_between(gate, timeline, start=split[0].changed_at, end=split[-1].changed_at)bucketing = bucketing_net_effect(history.counted(BucketingChange))- if bucketing and bucketing.shape == BucketingChangeShape.REVERTED:+ if (+ ExperimentHealthFlagChangeKind.BUCKETING in asked+ and bucketing+ and bucketing.shape == BucketingChangeShape.REVERTED+ ):exposed_between[ExperimentHealthFlagChangeKind.BUCKETING] = _exposed_between(gate, timeline, start=bucketing.first.changed_at, end=bucketing.last.changed_at)conditions = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)- if conditions and conditions.shape == ConditionsChangeShape.REVERTED:+ if (+ ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS in asked+ and conditions+ and conditions.shape == ConditionsChangeShape.REVERTED+ ):exposed_between[ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS] = _exposed_between(gate, timeline, start=conditions.first.changed_at, end=conditions.last.changed_at)diff --git a/products/experiments/backend/health/exposure_inputs.py b/products/experiments/backend/health/exposure_inputs.pyindex bbd3c6ecacf4..0acec2767e96 100644--- a/products/experiments/backend/health/exposure_inputs.py+++ b/products/experiments/backend/health/exposure_inputs.py@@ -88,6 +88,8 @@ def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_queryquery.exposure_criteria, experiment.team, experiment.start_date).multiple_variant_handling,sample_ratio_mismatch_p_value=srm.p_value if srm else None,+ sample_ratio_expected={},+ daily_exposures={},# The checks read the hours at computation, not now.hours_since_launch=(computed_at - experiment.start_date).total_seconds() / 3600 if computed_at else None,counts_groups=aggregates_by_group(runner.group_type_index),diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 1b1411713947..61b6cdfd5797 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -102,7 +102,8 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = ()# The exposure query runs these on its answer. They must not include the checks above: the exposure answer-# is cached for a day, and a flag edit would not clear their findings.+# is cached for a day, and a flag edit would not clear their findings. `srm` reads the verdict of+# `split_changed_mid_run`, which only a split edit can change, and the split is part of the exposure query's cache key.EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (HealthCheck(key=ExperimentHealthCheckKey.ZERO_EXPOSURES,diff --git a/products/experiments/backend/health/test/test_exposure_gate.py b/products/experiments/backend/health/test/test_exposure_gate.pyindex e889231b13f5..0ab7a201bd12 100644--- a/products/experiments/backend/health/test/test_exposure_gate.py+++ b/products/experiments/backend/health/test/test_exposure_gate.py@@ -29,6 +29,7 @@ from products.experiments.backend.health.exposure_gate import (MAX_EXPOSURE_QUERIES,MAX_OVERRIDE_WINDOWS,load_flag_history,+ load_split_history,query_any_exposure,query_exposed_units,)@@ -130,6 +131,7 @@ class TestLoadFlagHistory(BaseTest):*,units: list[int | Exception] | Callable[..., int] | None = None,reader_access: ExperimentHealthReaderAccess = FULL_ACCESS,+ load: Callable[..., FlagHistory | FlagHistoryNotChecked | None] = load_flag_history,) -> tuple[FlagHistory | FlagHistoryNotChecked | None, MagicMock]:with (patch("products.experiments.backend.health.exposure_gate.query_any_exposure", side_effect=answers) as query,@@ -139,7 +141,7 @@ class TestLoadFlagHistory(BaseTest):return_value=MIN_EXPOSED_UNITS_TO_WARN,) as units_query,):- history = load_flag_history(self.experiment, reader_access=reader_access)+ history = load(self.experiment, reader_access=reader_access)self.units_query = units_queryreturn history, query@@ -244,6 +246,27 @@ class TestLoadFlagHistory(BaseTest):],)+ def test_split_history_counts_only_the_units_before_the_last_split_change(self) -> None:+ first_at = self.launched_at + timedelta(days=1)+ self._log_change(first_at, key=("first-key", FLAG_KEY))+ self._log_split_changes([_split(50, 50), _split(60, 40)], first_at=first_at + timedelta(hours=1))++ history, _ = self._load([True], load=load_split_history)++ assert isinstance(history, FlagHistory)+ self.assertEqual(+ history.exposed_units_before_last, {ExperimentHealthFlagChangeKind.SPLIT: MIN_EXPOSED_UNITS_TO_WARN}+ )+ self.assertEqual(self.units_query.call_count, 1)++ def test_split_history_without_a_split_change_queries_no_exposures(self) -> None:+ self._log_change(self.launched_at + timedelta(days=1), bucketing_identifier=("distinct_id", "device_id"))++ history, query = self._load([True], load=load_split_history)++ assert isinstance(history, FlagHistory)+ self.assertEqual((query.call_count, self.units_query.call_count), (0, 0))+def test_a_count_is_cached_apart_from_the_exposure_answer_of_the_same_window(self) -> None:self._log_split_changes([_split(50, 50), _split(60, 40)])first, _ = self._load([True], units=[3])diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.pyindex 82a85f5e4b94..b91e81f62ee7 100644--- a/products/experiments/backend/health/test/test_registry.py+++ b/products/experiments/backend/health/test/test_registry.py@@ -37,6 +37,8 @@ UNEVEN_SPLIT_TOTALS = ExposureTotals(total_exposures={"control": 800, "test": 200, "$multiple": 20},multiple_variant_handling=MultipleVariantHandling.EXCLUDE,sample_ratio_mismatch_p_value=None,+ sample_ratio_expected={},+ daily_exposures={},hours_since_launch=72,counts_groups=False,)diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex edd7480bc924..f2e4f38219e2 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -1,11 +1,13 @@from dataclasses import replace-from datetime import UTC, datetime, timedelta+from datetime import UTC, date, datetime, timedeltafrom typing import Anyfrom unittest import TestCasefrom parameterized import parameterized+from posthog.schema import MultipleVariantHandling+from products.experiments.backend.facade.contracts import (ExperimentHealthFinding,ExperimentHealthFindingActionKind,@@ -16,7 +18,14 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readablefrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.split_changed import split_changed_mid_run-from products.experiments.backend.health.context import FlagHistoryNotCheckedReason, HealthContext, parse_flag_state+from products.experiments.backend.health.checks.srm import reads_flag_history, srm+from products.experiments.backend.health.context import (+ ExposureEvents,+ ExposureTotals,+ FlagHistoryNotCheckedReason,+ HealthContext,+ parse_flag_state,+)from products.experiments.backend.health.skips import (FLAG_DELETED,NEEDS_VARIANTS,@@ -472,3 +481,131 @@ class TestSplitChangedMidRun(TestCase):"with a conclusion, or adjust the variant distribution of the linked feature flag to resume proper A/B ""testing.",)+++def _totals(after_change: dict[str, int], *, counts_groups: bool = False) -> ExposureTotals:+ # Exposed under the 50/50 split on the day of the change, then the given ones on the next day under 80/20.+ before = {"control": 500, "test": 500}+ total_exposures = {variant: before[variant] + after_change[variant] for variant in before}+ whole_run = sum(total_exposures.values())+ return ExposureTotals(+ total_exposures=total_exposures,+ multiple_variant_handling=MultipleVariantHandling.EXCLUDE,+ sample_ratio_mismatch_p_value=0.00001,+ sample_ratio_expected={"control": 0.8 * whole_run, "test": 0.2 * whole_run},+ daily_exposures={+ variant: {FIRST_CHANGE_AT.date(): before[variant], date(2026, 9, 13): after_change[variant]}+ for variant in before+ },+ hours_since_launch=72,+ counts_groups=counts_groups,+ )+++MATCHES_THE_NEW_SPLIT = {"control": 400, "test": 100}+REVERSED_SPLIT = _split(("test", 20), ("control", 80))+++ACTIVATION_EVENTS = ExposureEvents(+ custom_event=None,+ required_flag_key="flag",+ variant_property="$feature_flag_response",+ flag_key_filters=(),+ activation="checkout",+)+PLAIN_TITLE = "Users are not split across variants as configured"+++class TestSrmAfterSplitChange(TestCase):+ @parameterized.expand(+ [+ (+ "retest_passes",+ _context(EVEN, UNEVEN, exposed_units=5),+ _totals(MATCHES_THE_NEW_SPLIT),+ (+ "split_changed",+ "The users first exposed after the day of the change match the current split (p = 1).",+ ),+ ),+ (+ "retest_fails",+ _context(EVEN, UNEVEN, exposed_units=5),+ _totals({"control": 250, "test": 250}),+ ("split_changed", "don't match the current split either (p < 0.001), so look for another cause too."),+ ),+ (+ "too_few_for_a_retest",+ _context(EVEN, UNEVEN, exposed_units=5),+ _totals({"control": 40, "test": 10}),+ ("split_changed", "Too few users were first exposed after the day of the change to test them alone."),+ ),+ (+ "activation_dates_users_after_their_flag_call",+ replace(_context(EVEN, UNEVEN, exposed_units=5), exposure_events=ACTIVATION_EVENTS),+ _totals(MATCHES_THE_NEW_SPLIT),+ ("split_changed", "so the users exposed after the change can't be tested alone."),+ ),+ (+ "groups",+ _context(EVEN, UNEVEN, exposed_units=5),+ _totals(MATCHES_THE_NEW_SPLIT, counts_groups=True),+ ("split_changed", "Groups are not split as configured after a split change"),+ ),+ (+ "reorder",+ _context(UNEVEN, REVERSED_SPLIT, exposed_units=5),+ _totals(MATCHES_THE_NEW_SPLIT),+ ("split_changed", "which can cause this."),+ ),+ (+ "change_before_4_people",+ _context(EVEN, UNEVEN, exposed_units=4),+ _totals(MATCHES_THE_NEW_SPLIT),+ (None, PLAIN_TITLE),+ ),+ (+ "history_not_checked",+ _context(EVEN, UNEVEN, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),+ _totals(MATCHES_THE_NEW_SPLIT),+ (None, PLAIN_TITLE),+ ),+ (+ "history_not_loaded",+ replace(_context(EVEN, UNEVEN, exposed_units=5), flag_history=None),+ _totals(MATCHES_THE_NEW_SPLIT),+ (None, PLAIN_TITLE),+ ),+ ]+ )+ def test_names_a_split_change_as_the_likely_cause(+ self, _name: str, ctx: HealthContext, totals: ExposureTotals, expected: tuple[str | None, str]+ ) -> None:+ outcome = srm(replace(ctx, exposures=totals))++ assert isinstance(outcome, ExperimentHealthFinding)+ subcode, text = expected+ self.assertEqual((outcome.subcode, outcome.severity), (subcode, ExperimentHealthFindingSeverity.WARNING))+ self.assertIn(text, f"{outcome.title} {outcome.detail}")++ @parameterized.expand([("failing_test", 0.00001, True), ("passing_test", 0.01, False)])+ def test_reads_the_flag_history_only_for_a_mismatch(self, _name: str, p_value: float, expected: bool) -> None:+ ctx = replace(+ RUNNING_CONTEXT,+ exposures=replace(_totals(MATCHES_THE_NEW_SPLIT), sample_ratio_mismatch_p_value=p_value),+ )++ self.assertEqual(reads_flag_history(ctx), expected)++ def test_names_the_cause_the_evidence_and_the_next_step(self) -> None:+ outcome = srm(replace(_context(EVEN, UNEVEN, exposed_units=5), exposures=_totals(MATCHES_THE_NEW_SPLIT)))++ assert isinstance(outcome, ExperimentHealthFinding)+ self.assertEqual(outcome.title, "Users are not split as configured after a split change")+ self.assertEqual(+ outcome.detail,+ "The distribution of users across variants doesn't match your current rollout percentages (p < 0.001). "+ "The variant split changed after launch while users were exposed, and some of them got their variants "+ "from another split, which can cause this. The users first exposed after the day of the change match the "+ 'current split (p = 1). Follow the steps in "The variant split changed after launch" to fix the results.',+ )diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.pyindex 49bc63529691..bd64b701592f 100644--- a/products/experiments/backend/health/test/test_zero_exposures.py+++ b/products/experiments/backend/health/test/test_zero_exposures.py@@ -40,6 +40,8 @@ NO_EXPOSURES = ExposureTotals(total_exposures={"control": 0, "test": 0},multiple_variant_handling=MultipleVariantHandling.EXCLUDE,sample_ratio_mismatch_p_value=None,+ sample_ratio_expected={},+ daily_exposures={},hours_since_launch=48,counts_groups=False,)diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex ddb68f41bc12..e238ca56eb7d 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -1,9 +1,9 @@import uuid-from datetime import UTC, datetime, timedelta+from dataclasses import replace+from datetime import UTC, date, datetime, timedeltaimport structlogfrom rest_framework.exceptions import ValidationError-from scipy.stats import chisquarefrom posthog.schema import (BiasRisk,@@ -28,15 +28,29 @@ from posthog.hogql import astfrom posthog.hogql.constants import HogQLGlobalSettingsfrom posthog.hogql.query import execute_hogql_query-from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context+from posthog.clickhouse.query_tagging import Product, get_query_tags, tag_queries, tags_context+from posthog.exceptions_capture import capture_exceptionfrom posthog.hogql_queries.query_runner import QueryRunnerfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.models.team.extensions import get_or_create_team_extension+from posthog.models.user import User+from products.access_control.backend.facade.api import (+ get_restricted_properties_with_group_type_index_for_team,+ split_restricted_property_names,+)+from products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult-from products.experiments.backend.facade.contracts import ExperimentHealth+from products.experiments.backend.facade.contracts import ExperimentHealth, ExperimentHealthReaderAccessfrom products.experiments.backend.health.checks.bias_risk import evaluate_bias_risk-from products.experiments.backend.health.context import ExposureTotals, load_health_context+from products.experiments.backend.health.checks.srm import reads_flag_history, sample_ratio_test+from products.experiments.backend.health.context import (+ ExposureTotals,+ FlagHistory,+ FlagHistoryNotChecked,+ load_health_context,+)+from products.experiments.backend.health.exposure_gate import load_split_historyfrom products.experiments.backend.health.registry import (EXPOSURE_HEALTH_CHECKS,evaluate as evaluate_health,@@ -60,12 +74,12 @@ from products.experiments.backend.models.team_experiments_config import TeamExpelogger = structlog.get_logger(__name__)QUERY_ROW_LIMIT = 5000 # The result has one row per (day, variant) pair-SRM_MINIMUM_SAMPLE_SIZE = 100 # Minimum total exposures required for SRM calculationdef build_experiment_exposure_query(experiment: Experiment) -> ExperimentExposureQuery:- """The exposure query of an experiment over its own window. Backend callers build it here, so they share one- cache entry. The page builds its own, with the whole flag, so it caches apart."""+ """The exposure query of an experiment over its own window. Backend callers build it here, so callers with the same+ access to the flag's history share one cache entry. The page builds its own, with the whole flag, so it caches+ apart."""feature_flag = experiment.feature_flag# The FK gives a model instance, and the query field is a pydantic type with no# attribute-based validation, so it must be serialized first.@@ -94,6 +108,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):super().__init__(*args, **kwargs)# Same contract as ExperimentQueryRunner.__init__: tags the terminal error event, and None keeps it silent.self.error_event_context = error_event_context+ # The user id and whether that user may read the flag's history.+ self._flag_history_access: tuple[int, bool] | None = Noneif not self.query.experiment_id:raise ValidationError("experiment_id is required")@@ -248,46 +264,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):# excluded_variants are not in the exposure chart, so they must not enter the chi-square.rollout_percentages = {k: v for k, v in rollout_percentages.items() if k not in self.excluded_variants}- # Iterate over the variants with rollout, not total_exposures, so that a variant with 0- # exposures still enters the chi-square and sum(observed) == sum(expected).- variants_with_rollout = {key for key, pct in rollout_percentages.items() if pct > 0}-- total_observed = sum(- total_exposures.get(key, 0) for key in variants_with_rollout if key != MULTIPLE_VARIANT_KEY- )- if total_observed < SRM_MINIMUM_SAMPLE_SIZE:- return None-- # After dropping holdout and excluded variants, the remaining percentages may not sum to 100.- # Normalize so that the chi-square expected counts sum to total_observed.- total_rollout = sum(rollout_percentages[k] for k in variants_with_rollout if k != MULTIPLE_VARIANT_KEY)- if total_rollout <= 0:- return None-- observed: list[float] = []- expected: list[float] = []- expected_counts: dict[str, float] = {}-- for variant_key in variants_with_rollout:- if variant_key == MULTIPLE_VARIANT_KEY:- continue-- obs_count = total_exposures.get(variant_key, 0)- rollout_pct = rollout_percentages[variant_key]- exp_count = (rollout_pct / total_rollout) * total_observed-- observed.append(float(obs_count))- expected.append(exp_count)- expected_counts[variant_key] = exp_count-- if len(observed) < 2:- return None-- _, p_value = chisquare(observed, expected)-- return SampleRatioMismatch(- expected=expected_counts,- p_value=float(p_value),+ return sample_ratio_test(+ total_exposures,+ {key: pct for key, pct in rollout_percentages.items() if pct > 0 and key != MULTIPLE_VARIANT_KEY},)def _evaluate_bias_risk(@@ -308,8 +287,47 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):total_exposures=total_exposures,)+ def _can_read_flag_history(self) -> bool:+ # The flag's history stays behind the flag's own access controls and, for a token, the `activity_log:read`+ # scope, as on the health endpoint. A query runner can't see a token's scopes, so a request with any token reads+ # no history, and neither does a userless run or a principal that is not a user.+ if not isinstance(self.user, User) or get_query_tags().access_method is not None:+ return False+ # The cache key asks on every run, and each check loads the project's access rules.+ if self._flag_history_access is None or self._flag_history_access[0] != self.user.pk:+ try:+ readable = UserAccessControl(user=self.user, team=self.team).check_access_level_for_object(+ self.experiment.feature_flag, required_level="viewer"+ )+ except Exception as error:+ # The answer feeds the cache key, so a failure here would fail the exposure query.+ capture_exception(error)+ readable = False+ self._flag_history_access = (self.user.pk, readable)+ return self._flag_history_access[1]++ def _flag_history(self) -> FlagHistory | FlagHistoryNotChecked | None:+ try:+ restrictions = get_restricted_properties_with_group_type_index_for_team(+ user=self.user if isinstance(self.user, User) else None, team_id=self.team.pk+ )+ except Exception as error:+ # The exposure answer must not fail for the history that only explains one finding.+ capture_exception(error)+ return None+ reader_access = ExperimentHealthReaderAccess(+ can_read_flag_history=self._can_read_flag_history(),+ # The reader is already querying the exposure events.+ can_query_events=True,+ denied_event_properties=split_restricted_property_names(restrictions).event,+ )+ return load_split_history(self.experiment, reader_access=reader_access)+def _evaluate_health(self, exposures: ExposureTotals) -> ExperimentHealth:- return evaluate_health(load_health_context(self.experiment, exposures), EXPOSURE_HEALTH_CHECKS)+ context = load_health_context(self.experiment, exposures)+ if reads_flag_history(context):+ context = replace(context, flag_history=self._flag_history())+ return evaluate_health(context, EXPOSURE_HEALTH_CHECKS)@staticmethoddef _health_findings(health: ExperimentHealth) -> list[ExperimentExposureHealthFinding]:@@ -371,11 +389,13 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):variant_series: dict[str, ExperimentExposureTimeSeries] = {}variant_data: dict[str, dict[str, int]] = {}+ daily_exposures: dict[str, dict[date, int]] = {}for result in response.results:day, variant, count = resultif variant not in variant_data:variant_data[variant] = {}variant_data[variant][day.isoformat()] = count+ daily_exposures.setdefault(variant, {})[day] = int(count)for variant, daily_counts in variant_data.items():sorted_days = sorted(daily_counts.keys())@@ -413,6 +433,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):total_exposures=total_exposures,multiple_variant_handling=multiple_variant_handling,sample_ratio_mismatch_p_value=sample_ratio_mismatch.p_value if sample_ratio_mismatch else None,+ sample_ratio_expected=sample_ratio_mismatch.expected if sample_ratio_mismatch else {},+ daily_exposures=daily_exposures,hours_since_launch=self.experiment.hours_since_launch,counts_groups=aggregates_by_group(self.group_type_index),)@@ -448,8 +470,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):result_dict = {}variants = set(self.variants)- for date, variant, count in results:- result_dict[(date, variant)] = count+ for day, variant, count in results:+ result_dict[(day, variant)] = countvariants.add(variant)complete_results = []@@ -465,4 +487,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):def get_cache_payload(self) -> dict:payload = super().get_cache_payload()payload["experiment_exposures_response_version"] = 2+ # The sample ratio finding can name a split change from the flag's history, so a reader who can't read the+ # history, a userless run included, must not get a cached answer of one who can.+ if not self._can_read_flag_history():+ payload["flag_history_hidden"] = Truereturn payloaddiff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex 60a53315356f..09ac44586d26 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -19,8 +19,13 @@ from posthog.schema import (MultipleVariantHandling,)+from posthog.clickhouse.query_tagging import AccessMethod, tags_context+from posthog.constants import AvailableFeature+from posthog.models.activity_logging.activity_log import ActivityLog+from posthog.models.user import Userfrom posthog.test.test_journeys import journeys_for+from products.access_control.backend.models.access_control import AccessControlfrom products.actions.backend.models.action import Actionfrom products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEYfrom products.experiments.backend.hogql_queries.experiment_exposures_query_runner import ExperimentExposuresQueryRunner@@ -1481,6 +1486,106 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):self.assertEqual(response.sample_ratio_mismatch.expected["control"], 50.0)self.assertEqual(response.sample_ratio_mismatch.expected["test"], 50.0)+ def _running_exposure_query(self) -> ExperimentExposureQuery:+ self.experiment.end_date = None+ self.experiment.save()+ return ExperimentExposureQuery(+ kind="ExperimentExposureQuery",+ experiment_id=self.experiment.id,+ experiment_name=self.experiment.name,+ feature_flag=model_to_dict(self.feature_flag),+ start_date=self.experiment.start_date.isoformat(),+ end_date=None,+ exposure_criteria=self.experiment.exposure_criteria,+ )++ @parameterized.expand([("reader_of_the_flag", True, "split_changed"), ("userless_run", False, None)])+ @time_machine.travel("2024-01-07T12:00:00Z", tick=False)+ def test_srm_names_a_split_change_for_a_reader_of_the_flag_history(+ self, _name: str, as_user: bool, subcode: str | None+ ) -> None:+ ff_property = f"$feature/{self.feature_flag.key}"+ # 90/10 before the change on 2024-01-03, and the current 50/50 split after it.+ journeys_for(+ {+ f"user_{day}_{variant}_{i}": [+ {+ "event": "$feature_flag_called",+ "timestamp": day,+ "properties": {+ "$feature_flag_response": variant,+ ff_property: variant,+ "$feature_flag": self.feature_flag.key,+ },+ }+ ]+ for day, counts in (+ ("2024-01-02", (("control", 90), ("test", 10))),+ ("2024-01-04", (("control", 60), ("test", 60))),+ )+ for variant, count in counts+ for i in range(count)+ },+ self.team,+ )+ flush_persons_and_events()+ before = {+ **self.feature_flag.filters,+ "multivariate": {+ "variants": [{"key": "control", "rollout_percentage": 90}, {"key": "test", "rollout_percentage": 10}]+ },+ }+ ActivityLog.objects.create(+ team_id=self.team.id,+ scope="FeatureFlag",+ item_id=str(self.feature_flag.id),+ activity="updated",+ detail={+ "changes": [+ {+ "type": "FeatureFlag",+ "action": "changed",+ "field": "filters",+ "before": before,+ "after": self.feature_flag.filters,+ }+ ]+ },+ created_at=datetime(2024, 1, 3, tzinfo=UTC),+ )++ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None+ ).calculate()++ self.assertEqual(+ [(finding.code, finding.subcode) for finding in response.health_findings or []],+ [(ExperimentExposureHealthFindingCode.SRM, subcode)],+ )++ def test_cache_key_keeps_the_flag_history_from_a_reader_who_cannot_view_the_flag(self) -> None:+ self.organization.available_product_features = [+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}+ ]+ self.organization.save()+ reader = User.objects.create_and_join(self.organization, "[email protected]", "password")+ query = self._running_exposure_query()++ def cache_key(user: User | None) -> str:+ return ExperimentExposuresQueryRunner(team=self.team, query=query, user=user).get_cache_key()++ visible_key = cache_key(reader)+ with tags_context(access_method=AccessMethod.PERSONAL_API_KEY):+ token_key = cache_key(reader)+ AccessControl.objects.create(+ team=self.team, resource="feature_flag", resource_id=str(self.feature_flag.id), access_level="none"+ )+ hidden_key = cache_key(reader)++ self.assertNotEqual(hidden_key, visible_key)+ self.assertEqual(cache_key(None), hidden_key)+ self.assertEqual(token_key, hidden_key)+@parameterized.expand([("first_day", "2024-01-01T23:00:00Z", None, []),diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex bf0f6cf6a974..a7ae0e72c2f7 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -310,7 +310,11 @@ function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {return [exposureCheck('zero_exposures', ['experiment', 'flag', 'exposures'], null),- exposureCheck('srm', ['exposures'], 'The finding needs p < 0.001.'),+ exposureCheck(+ 'srm',+ ['exposures', 'flag'],+ 'The finding needs p < 0.001. After a split change it tests the users first exposed after the day of the change. It reads the flag history only for a reader who can view the flag.'+ ),exposureCheck('bias_risk_multiple_excluded', ['exposures'], null),exposureCheck('multiple_variants_share',diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex 73ec2da0fdf1..3380a836e12d 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -242,8 +242,8 @@ export function captureExperimentHealthChecked(* every load. `experiment results refresh completed` follows the exposure load on every tab.* `has_srm`, `has_bias_risk` and a zero `exposures_total` are the page's own rules. The `_server` values are* the server's checks on the same answer, null when the answer holds no server findings (a cached answer- * from before the server ran them). `zero_exposures_cause` is the server's zero-exposures subcode, null without the- * finding or when it names no cause.+ * from before the server ran them). `zero_exposures_cause` and `srm_cause` are the server's subcodes of those findings,+ * null without the finding or when it names no cause.*/export function exposureHealthEventProperties(exposures: ExperimentExposureQueryResponse | null | undefined,@@ -254,6 +254,7 @@ export function exposureHealthEventProperties(has_srm: boolean | nullhas_bias_risk: boolean | nullsrm_server: boolean | null+ srm_cause: string | nullzero_exposures_server: boolean | nullzero_exposures_cause: string | nullbias_risk_server: boolean | null@@ -265,6 +266,7 @@ export function exposureHealthEventProperties(has_srm: null,has_bias_risk: null,srm_server: null,+ srm_cause: null,zero_exposures_server: null,zero_exposures_cause: null,bias_risk_server: null,@@ -282,6 +284,8 @@ export function exposureHealthEventProperties(has_srm: hasSampleRatioMismatch(exposures),has_bias_risk: exposures.bias_risk != null,srm_server: serverCodes ? serverCodes.has('srm') : null,+ // pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.+ srm_cause: exposures.health_findings?.find(({ code }) => code === 'srm')?.subcode ?? null,zero_exposures_server: serverCodes ? serverCodes.has('zero_exposures') : null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.zero_exposures_cause: exposures.health_findings?.find(({ code }) => code === 'zero_exposures')?.subcode ?? null,diff --git a/products/experiments/frontend/health/exposureHealth.test.ts b/products/experiments/frontend/health/exposureHealth.test.tsnew file mode 100644index 000000000000..cfc795558c47--- /dev/null+++ b/products/experiments/frontend/health/exposureHealth.test.ts@@ -0,0 +1,73 @@+import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'++import { SAMPLE_RATIO_MISMATCH_DESCRIPTION, sampleRatioLine } from './exposureHealth'++const srmFinding = (+ subcode: string | null+): NonNullable<ExperimentExposureQueryResponse['health_findings']>[number] => ({+ code: 'srm',+ subcode,+ severity: 'warning',+ title: 'Users are not split as configured',+ detail: `Server detail for ${subcode ?? 'a mismatch'}`,+ evidence: {},+ actions: [],+ diagnostic_ref: 'A2',+})++const MISMATCH = {+ kind: NodeKind.ExperimentExposureQuery,+ timeseries: [],+ total_exposures: { control: 600, test: 400 },+ date_range: { date_from: '2026-10-01', date_to: '2026-10-02' },+ sample_ratio_mismatch: { expected: { control: 500, test: 500 }, p_value: 0.00001 },+ health_findings: [srmFinding('split_changed')],+ health_checks: [],+} satisfies ExperimentExposureQueryResponse++describe('sampleRatioLine', () => {+ it.each([+ {+ desc: 'a mismatch after a split change',+ exposures: MISMATCH,+ followsServer: true,+ expected: {+ tone: 'warning',+ label: 'Sample ratio mismatch after a split change',+ tooltip: 'Server detail for split_changed',+ },+ },+ {+ desc: 'a mismatch the server reports without a cause',+ exposures: { ...MISMATCH, health_findings: [srmFinding(null)] },+ followsServer: true,+ expected: {+ tone: 'warning',+ label: 'Sample ratio mismatch detected',+ tooltip: 'Server detail for a mismatch',+ },+ },+ {+ desc: 'a reader without the health panel',+ exposures: MISMATCH,+ followsServer: false,+ expected: {+ tone: 'warning',+ label: 'Sample ratio mismatch detected',+ tooltip: SAMPLE_RATIO_MISMATCH_DESCRIPTION,+ },+ },+ {+ desc: 'an answer cached before the server ran its checks',+ exposures: { ...MISMATCH, health_findings: undefined },+ followsServer: true,+ expected: {+ tone: 'warning',+ label: 'Sample ratio mismatch detected',+ tooltip: SAMPLE_RATIO_MISMATCH_DESCRIPTION,+ },+ },+ ])('follows the server only with the health panel: $desc', ({ exposures, followsServer, expected }) => {+ expect(sampleRatioLine(exposures, followsServer)).toMatchObject(expected)+ })+})diff --git a/products/experiments/frontend/health/exposureHealth.ts b/products/experiments/frontend/health/exposureHealth.tsindex e113f72c91c9..4de5e21f764a 100644--- a/products/experiments/frontend/health/exposureHealth.ts+++ b/products/experiments/frontend/health/exposureHealth.ts@@ -1,4 +1,4 @@-import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'+import type { ExperimentExposureHealthFinding, ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'const SAMPLE_RATIO_MISMATCH_P_VALUE = 0.001@@ -12,6 +12,55 @@ export function hasSampleRatioMismatch(exposures: ExperimentExposureQueryRespons)}+export const SAMPLE_RATIO_MATCH_DESCRIPTION =+ 'No sample ratio mismatch detected. The difference between actual and expected exposures is within normal random variation.'++/** The server's sample ratio finding in the exposure answer, or undefined when the server reports none. */+export function serverSampleRatioFinding(+ exposures: ExperimentExposureQueryResponse | null | undefined+): ExperimentExposureHealthFinding | undefined {+ return exposures?.health_findings?.find(({ code }) => code === 'srm')+}++/** What the exposures card says about the sample ratio test. */+export type SampleRatioLine = {+ tone: 'warning' | 'success'+ label: string+ tooltip: string+}++/**+ * The exposures card's line about the sample ratio test, or null without a test. With the health panel the card uses+ * the server's finding, which can name a split change as the likely cause. Without it the card keeps the page's own+ * rule and text.+ */+export function sampleRatioLine(+ exposures: ExperimentExposureQueryResponse | null | undefined,+ followsServer: boolean+): SampleRatioLine | null {+ if (exposures?.sample_ratio_mismatch == null) {+ return null+ }+ const finding = followsServer ? serverSampleRatioFinding(exposures) : undefined+ if (finding) {+ return {+ tone: 'warning',+ label:+ finding.subcode === 'split_changed'+ ? 'Sample ratio mismatch after a split change'+ : 'Sample ratio mismatch detected',+ tooltip: finding.detail,+ }+ }+ return hasSampleRatioMismatch(exposures)+ ? { tone: 'warning', label: 'Sample ratio mismatch detected', tooltip: SAMPLE_RATIO_MISMATCH_DESCRIPTION }+ : {+ tone: 'success',+ label: 'Exposure distribution matches rollout percentages',+ tooltip: SAMPLE_RATIO_MATCH_DESCRIPTION,+ }+}+/** The exposed users of every variant in the exposure answer, the `$multiple` variant included. */export function getTotalExposures(exposures: ExperimentExposureQueryResponse | null | undefined): number {return (exposures?.timeseries ?? []).reduce(diff --git a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.mdindex 64d1fa3edde7..a809673815f3 100644--- a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md+++ b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md@@ -127,8 +127,9 @@ That holds for an even split, and a user may quote it: the banner is the productMUST read the product's value.MUST NOT run the test on event counts of your own: raw exposure events include test accounts, responses that are no variant, repeated events of one person and events outside the analysis window.-<!-- Source for maintainers (may rot): _calculate_srm and SRM_MINIMUM_SAMPLE_SIZE in-products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py; hasSampleRatioMismatch in+<!-- Source for maintainers (may rot): _calculate_srm in+products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py; sample_ratio_test and+SRM_MINIMUM_SAMPLE_SIZE in products/experiments/backend/health/checks/srm.py; hasSampleRatioMismatch inproducts/experiments/frontend/health/exposureHealth.ts. -->**What it means.** The distribution of exposed people differs from the flag's current split by more than chance allows.