Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): mark health checks that can't judge as not applicable (#128085)

Marcel Poelkerc97655996c38
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
index 797082285855..61db28d3a99a 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
@@ -67,7 +67,7 @@ const EXPOSURE_CHECKS: Check[] = [
{
...passed('bias_risk_multiple_excluded', 'Bias from an uneven split'),
status: 'not_applicable',
- reason: 'The split is even',
+ reason: 'Runs when the variant split is uneven',
},
passed('multiple_variants_share', 'Users exposed to multiple variants'),
]
diff --git a/products/experiments/backend/health/checks/bias_risk.py b/products/experiments/backend/health/checks/bias_risk.py
index 8d88a98a20d8..c471cc5f0808 100644
--- a/products/experiments/backend/health/checks/bias_risk.py
+++ b/products/experiments/backend/health/checks/bias_risk.py
@@ -9,13 +9,22 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthMeasurement,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
)
+from products.experiments.backend.health.checks.forced_variant import assigns_nobody_by_split
from products.experiments.backend.health.context import HealthContext
from products.experiments.backend.health.skips import (
+ NEEDS_EXPOSURES,
+ NEEDS_SPLIT_VARIANTS,
NO_FLAG,
NOT_AFTER_END,
RUNS_AFTER_LAUNCH,
+ SPLIT_ASSIGNS_NOBODY,
CheckOutcome,
+ CheckPassed,
+ CheckSkip,
not_applicable,
)
from products.feature_flags.backend.facade.edits import FlagVariant
@@ -29,20 +38,15 @@ MULTIPLE_VARIANT_BIAS_THRESHOLD = 0.1 # on the 0-100 scale (0.1 = 0.1%)
MULTIPLE_VARIANT_BIAS_MINIMUM_COUNT = 5
-def evaluate_bias_risk(
+def _multiple_variant_share(
variants: Sequence[FlagVariant],
excluded_variant_keys: Collection[str],
multiple_variant_handling: MultipleVariantHandling,
total_exposures: Mapping[str, int],
-) -> BiasRisk | None:
- """
- Empirically observed multi-variant exclusion bias risk: uneven shares of the variants that the flag serves and the
- analysis compares + EXCLUDE handling + observed `$multiple` share above the threshold, from enough users.
- Returns a `BiasRisk` only when all three conditions hold; `None` otherwise.
- """
+) -> float | CheckSkip:
+ """The share of exposed users seen in multiple variants, on the 0-100 scale, or why the risk can't arise."""
if multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
- return None
-
+ return not_applicable('Runs when "Multiple variant handling" is "Exclude from analysis"')
# Excluded variants don't count, because the exposure query drops their events before it counts users in several
# variants. `is_evenly_distributed` needs shares that add up to 100. Shares one point apart are even, like 34/33/33.
compared = [
@@ -52,18 +56,35 @@ def evaluate_bias_risk(
and variant.rollout_percentage > 0
and variant.key not in excluded_variant_keys
]
- if len(compared) < 2 or max(compared) - min(compared) <= 1:
- return None
+ if len(compared) < 2:
+ return not_applicable(NEEDS_SPLIT_VARIANTS)
+ if max(compared) - min(compared) <= 1:
+ return not_applicable("Runs when the variant split is uneven")
+ total_observed = sum(total_exposures.values())
+ if total_observed <= 0:
+ return not_applicable(NEEDS_EXPOSURES)
+ return total_exposures.get(MULTIPLE_VARIANT_KEY, 0) / total_observed * 100
- multiple_observed = total_exposures.get(MULTIPLE_VARIANT_KEY, 0)
- if multiple_observed < MULTIPLE_VARIANT_BIAS_MINIMUM_COUNT:
- return None
- multiple_variant_percentage = (multiple_observed / sum(total_exposures.values())) * 100
- if multiple_variant_percentage <= MULTIPLE_VARIANT_BIAS_THRESHOLD:
- return None
+def _too_few_to_warn(total_exposures: Mapping[str, int]) -> bool:
+ return total_exposures.get(MULTIPLE_VARIANT_KEY, 0) < MULTIPLE_VARIANT_BIAS_MINIMUM_COUNT
+
- return BiasRisk(multiple_variant_percentage=multiple_variant_percentage)
+def evaluate_bias_risk(
+ variants: Sequence[FlagVariant],
+ excluded_variant_keys: Collection[str],
+ multiple_variant_handling: MultipleVariantHandling,
+ total_exposures: Mapping[str, int],
+) -> BiasRisk | None:
+ """
+ Empirically observed multi-variant exclusion bias risk: uneven shares of the variants that the flag serves and the
+ analysis compares + EXCLUDE handling + observed `$multiple` share above the threshold, from enough users.
+ Returns a `BiasRisk` only when all three conditions hold; `None` otherwise.
+ """
+ share = _multiple_variant_share(variants, excluded_variant_keys, multiple_variant_handling, total_exposures)
+ if isinstance(share, CheckSkip) or share <= MULTIPLE_VARIANT_BIAS_THRESHOLD or _too_few_to_warn(total_exposures):
+ return None
+ return BiasRisk(multiple_variant_percentage=share)
def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
@@ -73,13 +94,28 @@ def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
return not_applicable(NO_FLAG)
if ctx.has_ended:
return not_applicable(NOT_AFTER_END)
- risk = evaluate_bias_risk(
+ # The split's shares then say nothing about the variants' sizes. `forced_variant_release_condition` explains it.
+ if assigns_nobody_by_split(ctx.flag):
+ return not_applicable(SPLIT_ASSIGNS_NOBODY)
+ share = _multiple_variant_share(
variants=ctx.flag.variants,
excluded_variant_keys=ctx.excluded_variant_keys,
multiple_variant_handling=ctx.exposures.multiple_variant_handling,
total_exposures=ctx.exposures.total_exposures,
)
- if risk is None:
+ if isinstance(share, CheckSkip):
+ return share
+ if share <= MULTIPLE_VARIANT_BIAS_THRESHOLD:
+ return CheckPassed(
+ measurement=ExperimentHealthMeasurement(
+ value=share,
+ threshold=MULTIPLE_VARIANT_BIAS_THRESHOLD,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.ABOVE,
+ )
+ )
+ # The minimum count decides this pass, not the threshold, so it shows no measurement.
+ if _too_few_to_warn(ctx.exposures.total_exposures):
return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.BIAS_RISK_MULTIPLE_EXCLUDED,
@@ -87,12 +123,12 @@ def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
severity=ExperimentHealthFindingSeverity.WARNING,
title="The uneven split may bias the results",
detail=(
- f"{risk.multiple_variant_percentage:.1f}% of users were exposed to multiple variants. "
+ f"{share:.1f}% of users were exposed to multiple variants. "
"With an uneven variant split and the Exclude handling, these users were dropped more often "
"from the smaller variant, so its metrics can be biased. Use an even split and control "
"exposure with the overall rollout, or switch the handling to First seen."
),
- evidence={"multiple_variant_percentage": risk.multiple_variant_percentage},
+ evidence={"multiple_variant_percentage": share},
actions=(
ExperimentHealthFindingActionKind.ADJUST_DISTRIBUTION,
ExperimentHealthFindingActionKind.USE_FIRST_SEEN_VARIANT,
diff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.py
index 910d1f9eb79a..10a0bd72e05e 100644
--- a/products/experiments/backend/health/checks/early_significance.py
+++ b/products/experiments/backend/health/checks/early_significance.py
@@ -16,6 +16,7 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.context import HealthContext
from products.experiments.backend.health.skips import (
+ NEEDS_EXPOSURES,
NEEDS_PRIMARY_METRIC,
NOT_AFTER_END,
NOT_ON_ARCHIVED,
@@ -64,7 +65,7 @@ def early_significance(ctx: HealthContext) -> CheckOutcome:
return not_applicable("Runs once the primary metric has results")
samples = _total_samples(result)
if samples < MIN_SAMPLES_FOR_RESULTS:
- return not_applicable("Runs once enough users are exposed")
+ return not_applicable(NEEDS_EXPOSURES)
control_rate = result.baseline.mean or 0
if not 0 < control_rate < 1:
return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")
diff --git a/products/experiments/backend/health/checks/filtered_exposures.py b/products/experiments/backend/health/checks/filtered_exposures.py
index 568744401a15..e3eb5ee2096e 100644
--- a/products/experiments/backend/health/checks/filtered_exposures.py
+++ b/products/experiments/backend/health/checks/filtered_exposures.py
@@ -10,6 +10,7 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.context import ExposureTotals, HealthContext
from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS
from products.experiments.backend.health.skips import (
+ NEEDS_EXPOSURES,
RUNS_AFTER_LAUNCH,
CheckOutcome,
CheckPassed,
@@ -21,7 +22,7 @@ from products.experiments.backend.health.skips import (
# The count without the filter scans the exposure events, so the check runs only on results that look starved.
FEW_EXPOSURES_LIMIT = 1_000
# A filter that leaves out the project's own team and testers removes a small share of a live experiment's users.
-# Keeping this share or less means that the filter matches only some users and leaves out everyone else.
+# Keeping less than this share means that the filter matches only some users and leaves out everyone else.
KEPT_SHARE_THRESHOLD = 10.0 # on the 0-100 scale (10.0 = 10%)
# While few users are exposed, the project's own team can be most of them, so a small kept share is expected.
MINIMUM_EXPOSED_WITHOUT_FILTER = 100
@@ -48,8 +49,8 @@ def exposures_removed_by_test_account_filter(ctx: HealthContext) -> CheckOutcome
)
counted = exposures.total
if exposed < MINIMUM_EXPOSED_WITHOUT_FILTER:
- return None
- if counted * 100 > exposed * KEPT_SHARE_THRESHOLD:
+ return not_applicable(NEEDS_EXPOSURES)
+ if counted * 100 >= exposed * KEPT_SHARE_THRESHOLD:
return CheckPassed(
measurement=ExperimentHealthMeasurement(
value=counted / exposed * 100,
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index 65d5db41bd38..daaa0e44ae0e 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -37,7 +37,7 @@ from products.experiments.backend.health.context import (
PauseHistory,
)
from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD
-from products.experiments.backend.health.skips import NO_FLAG, CheckOutcome, not_applicable
+from products.experiments.backend.health.skips import NO_FLAG, CheckOutcome, CheckSkip, not_applicable
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
describe_split,
@@ -77,6 +77,7 @@ _MIN_PEOPLE_ASSIGNED_OUTSIDE = 10
_OUTSIDE_ASSIGNMENT_SETTLED_AFTER = timedelta(days=1)
_NOT_RUNNING_BUT_EXPOSED_TITLE = "The experiment is not running, but users are exposed to multiple variants"
+OTHER_EXPERIMENT_DECIDES = "Another experiment on the same feature flag decides the flag's state"
def flag_state(ctx: HealthContext) -> CheckOutcome:
@@ -140,7 +141,7 @@ def flag_state(ctx: HealthContext) -> CheckOutcome:
elif not ctx.archived:
# The draft state is also reached again when resetting experiment measurements.
if _running_others(ctx):
- return None
+ return not_applicable(OTHER_EXPERIMENT_DECIDES)
if flag.active and _has_multiple_variants_active(flag) and not zero_rollout:
# Info, because a flag turned on to test the variants before launch is the usual path.
return _finding(
@@ -248,11 +249,11 @@ def _no_rollout(*, observed: bool) -> ExperimentHealthFinding:
)
-def _live_after_end(ctx: HealthContext, ended_at: datetime) -> ExperimentHealthFinding | None:
+def _live_after_end(ctx: HealthContext, ended_at: datetime) -> ExperimentHealthFinding | CheckSkip:
others = ctx.other_experiments_on_flag
# The flag's state belongs to the later experiment, and that experiment's own check reports it.
if any(_serves_later_experiment(other, ended_at) for other in others):
- return None
+ return not_applicable(OTHER_EXPERIMENT_DECIDES)
if any(other.is_draft and not other.archived for other in others):
return _shared_with_draft(ended_at, ctx.team_timezone)
if ctx.flag_called_after_end is True:
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index df64e9b69dc6..52a95488ae7d 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -36,7 +36,15 @@ from products.experiments.backend.health.context import (
ReachableCondition,
)
from products.experiments.backend.health.pinned_share import pinned_variant_keys
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, CheckPassed, not_applicable
+from products.experiments.backend.health.skips import (
+ NEEDS_EXPOSURES,
+ NEEDS_SPLIT_VARIANTS,
+ RUNS_AFTER_LAUNCH,
+ SPLIT_ASSIGNS_NOBODY,
+ CheckOutcome,
+ CheckPassed,
+ not_applicable,
+)
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.feature_flags.backend.facade.edits import SplitChange
@@ -135,7 +143,7 @@ def srm(ctx: HealthContext) -> CheckOutcome:
# The split then serves nobody, so the test compares nothing. `forced_variant_release_condition` explains it.
overrides = _flag_of_the_run(ctx)
if overrides is not None and assigns_nobody_by_split(overrides):
- return not_applicable("The split assigns nobody, because every release condition sets a variant override")
+ return not_applicable(SPLIT_ASSIGNS_NOBODY)
p_value = exposures.sample_ratio_mismatch_p_value
if p_value is None:
# The exposure query compares only the variants with a share of the split that the analysis keeps, so after
@@ -146,8 +154,8 @@ def srm(ctx: HealthContext) -> CheckOutcome:
if variant.key and (variant.rollout_percentage or 0) > 0 and variant.key not in ctx.excluded_variant_keys
]
if len(variants_in_split) < 2:
- return not_applicable("Needs two or more variants in the split")
- return not_applicable("Runs once enough users are exposed")
+ return not_applicable(NEEDS_SPLIT_VARIANTS)
+ return not_applicable(NEEDS_EXPOSURES)
if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
return CheckPassed(
measurement=ExperimentHealthMeasurement(
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
index b8fb69bc699a..12e32c48d371 100644
--- a/products/experiments/backend/health/skips.py
+++ b/products/experiments/backend/health/skips.py
@@ -32,6 +32,9 @@ NOT_ON_ARCHIVED = "Doesn't run on archived experiments"
NO_FLAG = "The experiment has no feature flag"
FLAG_DELETED = "The feature flag is deleted"
NEEDS_VARIANTS = "Runs once the feature flag has variants"
+NEEDS_SPLIT_VARIANTS = "Needs two or more variants in the split"
+SPLIT_ASSIGNS_NOBODY = "The split assigns nobody, because every release condition sets a variant override"
+NEEDS_EXPOSURES = "Runs once enough users are exposed"
NEEDS_PRIMARY_METRIC = "Runs once the experiment has a primary metric"
diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.py
index c31c531774c5..fae80108bb73 100644
--- a/products/experiments/backend/health/test/test_flag_state.py
+++ b/products/experiments/backend/health/test/test_flag_state.py
@@ -15,7 +15,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
)
-from products.experiments.backend.health.checks.flag_state import flag_state
+from products.experiments.backend.health.checks.flag_state import OTHER_EXPERIMENT_DECIDES, flag_state
from products.experiments.backend.health.context import (
ConditionSetTraffic,
ExperimentOnFlag,
@@ -24,6 +24,7 @@ from products.experiments.backend.health.context import (
load_health_context,
parse_flag_state,
)
+from products.experiments.backend.health.skips import CheckSkip, not_applicable
from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status
from products.experiments.backend.models.experiment import Experiment
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -189,7 +190,7 @@ class TestFlagState(TestCase):
flag_called_after_end=True,
other_experiments=(_other(launched_at=ENDED_AT - timedelta(days=3)),),
),
- None,
+ not_applicable(OTHER_EXPERIMENT_DECIDES),
),
(
"ended_flag_served_a_later_experiment_that_ended",
@@ -202,7 +203,7 @@ class TestFlagState(TestCase):
_other(launched_at=ENDED_AT + timedelta(days=1), ended_at=ENDED_AT + timedelta(days=5)),
),
),
- None,
+ not_applicable(OTHER_EXPERIMENT_DECIDES),
),
(
"ended_flag_served_an_earlier_experiment",
@@ -258,7 +259,7 @@ class TestFlagState(TestCase):
filters=MULTIVARIANT,
other_experiments=(_other(launched_at=ENDED_AT - timedelta(days=3)),),
),
- None,
+ not_applicable(OTHER_EXPERIMENT_DECIDES),
),
("draft_zero_rollout", _context(DRAFT, active=True, filters=ZERO_ROLLOUT), None),
("draft_flag_disabled", _context(DRAFT, active=False, filters=MULTIVARIANT), None),
@@ -292,12 +293,15 @@ class TestFlagState(TestCase):
]
)
def test_flag_state(
- self, _name: str, ctx: HealthContext, expected: tuple[ExperimentHealthFindingCode, str, str | None] | None
+ self,
+ _name: str,
+ ctx: HealthContext,
+ expected: tuple[ExperimentHealthFindingCode, str, str | None] | CheckSkip | None,
) -> None:
finding = flag_state(ctx)
- if expected is None:
- self.assertIsNone(finding)
+ if not isinstance(expected, tuple):
+ self.assertEqual(finding, expected)
return
assert isinstance(finding, ExperimentHealthFinding)
code, subcode, variant_key = expected
@@ -537,12 +541,12 @@ class TestFlagState(TestCase):
class TestFlagStateOfAnEndedExperiment(BaseTest):
@parameterized.expand(
[
- ("running_experiment_on_the_flag", False, None),
+ ("running_experiment_on_the_flag", False, not_applicable(OTHER_EXPERIMENT_DECIDES)),
("deleted_experiment_on_the_flag", True, "ended_but_multiple_variants_rolled_out"),
]
)
def test_reads_the_other_experiments_on_the_flag(
- self, _name: str, other_deleted: bool, expected_subcode: str | None
+ self, _name: str, other_deleted: bool, expected: str | CheckSkip
) -> None:
flag = FeatureFlag.objects.create(
team=self.team,
@@ -572,4 +576,4 @@ class TestFlagStateOfAnEndedExperiment(BaseTest):
):
finding = flag_state(load_health_context(experiment))
- self.assertEqual(finding.subcode if isinstance(finding, ExperimentHealthFinding) else finding, expected_subcode)
+ self.assertEqual(finding.subcode if isinstance(finding, ExperimentHealthFinding) else finding, expected)
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index af1a3560ea78..930d7e358b03 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -27,6 +27,7 @@ from products.experiments.backend.health.context import (
FlagState,
HealthContext,
MetricEvents,
+ ReachableCondition,
)
from products.experiments.backend.health.registry import (
EXPERIMENT_HEALTH_CHECKS,
@@ -176,8 +177,8 @@ class TestEvaluate(TestCase):
[ExperimentHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES],
),
(
- "test_account_filter_keeps_10_percent_after_the_end",
- {**ENDED, "exposures": _filtered_totals(100, 1000)},
+ "test_account_filter_keeps_9_percent_after_the_end",
+ {**ENDED, "exposures": _filtered_totals(90, 1000)},
[ExperimentHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES],
),
("test_account_filter_keeps_95_percent", {"exposures": _filtered_totals(950, 1000)}, []),
@@ -240,7 +241,10 @@ class TestEvaluate(TestCase):
_totals_with_multiple(60, 1000), multiple_variant_handling=MultipleVariantHandling.FIRST_SEEN
)
},
- {Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_CHECKED, Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.PASSED},
+ {
+ Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_CHECKED,
+ Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.NOT_APPLICABLE,
+ },
),
(
"flag_traffic_is_left_to_zero_exposures_after_launch",
@@ -272,6 +276,34 @@ class TestEvaluate(TestCase):
_events("$$plugin_metrics", {}),
dict.fromkeys(EVENT_KEYS, Status.NOT_CHECKED),
),
+ (
+ "test_account_filter_waits_for_enough_exposures",
+ {"exposures": _filtered_totals(5, 60)},
+ {Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES: Status.NOT_APPLICABLE},
+ ),
+ (
+ "bias_risk_does_not_apply_to_an_even_split",
+ {"flag": EVEN_SPLIT_FLAG},
+ {Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.NOT_APPLICABLE},
+ ),
+ (
+ "bias_risk_waits_for_exposures",
+ {"exposures": NO_EXPOSURES},
+ {Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.NOT_APPLICABLE},
+ ),
+ (
+ "bias_risk_does_not_apply_when_the_split_assigns_nobody",
+ {
+ "flag": replace(
+ UNEVEN_FLAG,
+ reachable_conditions=(
+ ReachableCondition(number=1, pinned_variant="test", variant_property=None),
+ ),
+ ),
+ "exposures": _totals_with_multiple(60, 1000),
+ },
+ {Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.NOT_APPLICABLE},
+ ),
]
)
def test_check_statuses(self, _name: str, changes: dict[str, Any], expected: dict[Key, Status]) -> None:
@@ -386,9 +418,31 @@ class TestEvaluate(TestCase):
),
),
(
- "kept_share_below_the_minimum_exposed",
+ "kept_share_at_the_threshold",
Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
- _filtered_totals(5, 60),
+ _filtered_totals(100, 1000),
+ ExperimentHealthMeasurement(
+ value=10.0,
+ threshold=10.0,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.BELOW,
+ ),
+ ),
+ (
+ "bias_risk_share",
+ Key.BIAS_RISK_MULTIPLE_EXCLUDED,
+ _totals_with_multiple(0, 1000),
+ ExperimentHealthMeasurement(
+ value=0.0,
+ threshold=0.1,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.ABOVE,
+ ),
+ ),
+ (
+ "bias_risk_share_above_the_threshold_from_too_few_users",
+ Key.BIAS_RISK_MULTIPLE_EXCLUDED,
+ _totals_with_multiple(3, 1000),
None,
),
]