Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(experiments): let each health check return its own skip (#122360)

Marcel Poelker9de811a9c050
diff --git a/products/experiments/backend/health/checks/bias_risk.py b/products/experiments/backend/health/checks/bias_risk.py
index 9d60eb992617..d37d235ed42b 100644
--- a/products/experiments/backend/health/checks/bias_risk.py
+++ b/products/experiments/backend/health/checks/bias_risk.py
@@ -15,7 +15,7 @@ from products.experiments.backend.health.skips import (
NO_FLAG,
NOT_AFTER_END,
RUNS_AFTER_LAUNCH,
- CheckSkip,
+ CheckOutcome,
not_applicable,
)
from products.experiments.backend.variant_distribution import is_evenly_distributed
@@ -58,19 +58,13 @@ def evaluate_bias_risk(
return BiasRisk(multiple_variant_percentage=multiple_variant_percentage)
-def bias_risk_skip(ctx: HealthContext) -> CheckSkip | None:
+def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
if ctx.exposures is None:
return not_applicable(RUNS_AFTER_LAUNCH)
if ctx.flag is None:
return not_applicable(NO_FLAG)
if ctx.has_ended:
return not_applicable(NOT_AFTER_END)
- return None
-
-
-def bias_risk_multiple_excluded(ctx: HealthContext) -> ExperimentHealthFinding | None:
- if ctx.exposures is None or ctx.flag is None or ctx.has_ended:
- return None
risk = evaluate_bias_risk(
rollout_percentages=[variant.rollout_percentage for variant in ctx.flag.variants],
multiple_variant_handling=ctx.exposures.multiple_variant_handling,
diff --git a/products/experiments/backend/health/checks/bucketing_changed.py b/products/experiments/backend/health/checks/bucketing_changed.py
index 09d15fd71c61..a3dbdf02d500 100644
--- a/products/experiments/backend/health/checks/bucketing_changed.py
+++ b/products/experiments/backend/health/checks/bucketing_changed.py
@@ -22,15 +22,15 @@ from products.experiments.backend.health.bucketing_changes import (
from products.experiments.backend.health.checks.change_wording import duration, suggested_start, the_change
from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
- changes_not_checked,
+ changes_to_check,
delay_after_launch,
exposed_when,
few_exposed_units,
- history_to_check,
leave_out,
not_checked_finding,
)
from products.experiments.backend.health.context import FlagHistory, FlagHistoryNotChecked, HealthContext
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
from products.experiments.backend.health.split_changes import format_change_time
from products.feature_flags.backend.facade.edits import BucketingChange, BucketingSetting
@@ -47,10 +47,10 @@ _SETTING_NAMES = {
}
-def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
- history = history_to_check(ctx)
- if history is None:
- return None
+def bucketing_changed_mid_run(ctx: HealthContext) -> CheckOutcome:
+ history = changes_to_check(ctx)
+ if isinstance(history, CheckSkip):
+ return history
if isinstance(history, FlagHistoryNotChecked):
return _not_checked(history)
effect = bucketing_net_effect(history.counted(BucketingChange))
@@ -128,7 +128,7 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | N
def _not_checked(history: FlagHistoryNotChecked) -> ExperimentHealthFinding | None:
- effect = bucketing_net_effect(changes_not_checked(history, BucketingChange))
+ effect = bucketing_net_effect(history.of_kind(BucketingChange))
if effect is None:
return None
if effect.shape == BucketingChangeShape.REVERTED:
diff --git a/products/experiments/backend/health/checks/conditions_changed.py b/products/experiments/backend/health/checks/conditions_changed.py
index 1f9e2bc787b9..ce4386dc2660 100644
--- a/products/experiments/backend/health/checks/conditions_changed.py
+++ b/products/experiments/backend/health/checks/conditions_changed.py
@@ -23,10 +23,9 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.change_wording import suggested_start, the_change
from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
- changes_not_checked,
+ changes_to_check,
exposed_when,
few_exposed_units,
- history_to_check,
leave_out,
not_checked_finding,
)
@@ -51,6 +50,7 @@ from products.experiments.backend.health.pinned_share import (
pinned_conditions,
pinned_share_severity,
)
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
from products.experiments.backend.health.split_changes import format_change_time
from products.feature_flags.backend.facade.edits import ConditionsChange
@@ -61,10 +61,10 @@ _REVERTED_TITLE = "The release conditions changed and were set back"
_NOT_CHECKED_TITLE = "Release condition changes were not checked"
-def release_conditions_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
- history = history_to_check(ctx)
- if history is None:
- return None
+def release_conditions_changed_mid_run(ctx: HealthContext) -> CheckOutcome:
+ history = changes_to_check(ctx)
+ if isinstance(history, CheckSkip):
+ return history
if isinstance(history, FlagHistoryNotChecked):
return _not_checked(history)
effect = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
@@ -148,9 +148,9 @@ def _edited_overrides(effect: ConditionsNetEffect) -> list[str]:
def removed_overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
"""The overrides removed after launch, as pins on the conditions that held them, which the finding is graded
by as if they were still in place. None when no removed override has a condition left to count its users."""
- history = ctx.flag_history
+ history = changes_to_check(ctx)
flag = ctx.flag
- if not isinstance(history, FlagHistory) or flag is None or flag.deleted or not ctx.is_running or ctx.archived:
+ if not isinstance(history, FlagHistory) or flag is None:
return None
effect = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
if effect is None:
@@ -329,7 +329,7 @@ def _finding(
def _not_checked(history: FlagHistoryNotChecked) -> ExperimentHealthFinding | None:
- effect = conditions_net_effect(changes_not_checked(history, ConditionsChange), current=None)
+ effect = conditions_net_effect(history.of_kind(ConditionsChange), current=None)
if effect is None:
return None
return not_checked_finding(
diff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.py
index 85fa4f578fd4..a984e17c5a82 100644
--- a/products/experiments/backend/health/checks/early_significance.py
+++ b/products/experiments/backend/health/checks/early_significance.py
@@ -19,7 +19,7 @@ from products.experiments.backend.health.skips import (
NOT_AFTER_END,
NOT_ON_ARCHIVED,
RUNS_AFTER_LAUNCH,
- CheckSkip,
+ CheckOutcome,
not_applicable,
not_checked,
)
@@ -42,7 +42,7 @@ class _Comparison:
detectable: float
-def early_significance_skip(ctx: HealthContext) -> CheckSkip | None:
+def early_significance(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
if not ctx.is_launched:
@@ -61,22 +61,15 @@ def early_significance_skip(ctx: HealthContext) -> CheckSkip | None:
# "wait" after the page says the target is reached.
if metric.manual_running_time:
return not_applicable("Runs with the automatic running time estimate")
- if metric.result is None:
+ result = metric.result
+ if result is None:
return not_applicable("Runs once the primary metric has results")
- if _total_samples(metric.result) < MIN_SAMPLES_FOR_RESULTS:
+ samples = _total_samples(result)
+ if samples < MIN_SAMPLES_FOR_RESULTS:
return not_applicable("Runs once enough users are exposed")
- if not 0 < (metric.result.baseline.mean or 0) < 1:
- return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")
- return None
-
-
-def early_significance(ctx: HealthContext) -> ExperimentHealthFinding | None:
- metric = ctx.primary_metric
- if early_significance_skip(ctx) is not None or metric is None or metric.result is None:
- return None
- result = metric.result
control_rate = result.baseline.mean or 0
- samples = _total_samples(result)
+ if not 0 < control_rate < 1:
+ return not_applicable("Runs once the control group's conversion rate is above 0% and below 100%")
# The running time estimate counts the variants of the flag.
number_of_variants = len(ctx.flag.variants) if ctx.flag and ctx.flag.variants else len(result.variants) + 1
planned_samples = calculate_sample_size(
diff --git a/products/experiments/backend/health/checks/exposure_criteria.py b/products/experiments/backend/health/checks/exposure_criteria.py
index 02d14afdb2c3..f08606d1a8d6 100644
--- a/products/experiments/backend/health/checks/exposure_criteria.py
+++ b/products/experiments/backend/health/checks/exposure_criteria.py
@@ -20,7 +20,7 @@ from products.experiments.backend.health.skips import (
NO_FLAG,
NOT_AFTER_END,
NOT_ON_ARCHIVED,
- CheckSkip,
+ CheckOutcome,
not_applicable,
not_checked,
)
@@ -30,38 +30,25 @@ EXPOSURE_EVENT_NOT_INGESTED = "exposure_event_not_ingested"
FILTER_ON_ANOTHER_FLAG = "filter_on_another_flag"
-def exposure_criteria_skip(ctx: HealthContext) -> CheckSkip | None:
+def exposure_criteria_cannot_match(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
events = ctx.exposure_events
if events is None:
return not_checked("Couldn't check because the exposure criteria can't be read")
- if events.custom_event is not None:
+ event = events.custom_event
+ if event is not None:
if ctx.has_ended:
return not_applicable(NOT_AFTER_END)
- if events.custom_event not in ctx.events_received:
+ received = ctx.events_received.get(event)
+ if received is None:
return not_checked("Couldn't check because PostHog keeps no record of this event name")
- return None
- if events.required_flag_key is None:
- if ctx.flag is None:
- return not_applicable(NO_FLAG)
- return not_applicable("Doesn't check exposure criteria that use an action")
- return None
-
-
-def exposure_criteria_cannot_match(ctx: HealthContext) -> ExperimentHealthFinding | None:
- events = ctx.exposure_events
- if ctx.archived or events is None:
- return None
+ return None if received else _exposure_event_not_ingested(ctx, event)
flag_key = events.required_flag_key
- if flag_key is not None:
- other = next((f for f in events.flag_key_filters if _requires_another_flag(f, flag_key)), None)
- if other is not None:
- return _filter_on_another_flag(flag_key, other)
- event = events.custom_event
- if event is not None and not ctx.has_ended and ctx.events_received.get(event) is False:
- return _exposure_event_not_ingested(ctx, event)
- return None
+ if flag_key is None:
+ return not_applicable(NO_FLAG if ctx.flag is None else "Doesn't check exposure criteria that use an action")
+ other = next((f for f in events.flag_key_filters if _requires_another_flag(f, flag_key)), None)
+ return None if other is None else _filter_on_another_flag(flag_key, other)
def _requires_another_flag(flag_filter: FlagKeyFilter, flag_key: str) -> bool:
diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.py
index c68f063a88a0..6291aa38a068 100644
--- a/products/experiments/backend/health/checks/flag_change_findings.py
+++ b/products/experiments/backend/health/checks/flag_change_findings.py
@@ -24,7 +24,6 @@ from products.experiments.backend.health.checks.change_wording import (
)
from products.experiments.backend.health.context import (
MIN_EXPOSED_UNITS_TO_WARN,
- FlagChange,
FlagHistory,
FlagHistoryNotChecked,
FlagHistoryNotCheckedReason,
@@ -32,8 +31,9 @@ from products.experiments.backend.health.context import (
)
from products.experiments.backend.health.skips import (
NEEDS_VARIANTS,
+ CheckOutcome,
CheckSkip,
- flag_setup_skip,
+ flag_to_check,
not_applicable,
not_checked,
)
@@ -72,44 +72,36 @@ class ChangeEffect(Protocol):
def change_count(self) -> int: ...
-def history_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked | None:
- """The flag history that the flag-change checks read, or None when they don't apply."""
- history = ctx.flag_history
- flag = ctx.flag
- if history is None or flag is None or flag.deleted or not ctx.is_running or ctx.archived:
- return None
+def history_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked | CheckSkip:
+ """The flag history that the flag-history checks read, or why they don't look, in the order the reader is most
+ likely to act on."""
+ flag = flag_to_check(ctx, before_launch=False)
+ if isinstance(flag, CheckSkip):
+ return flag
# `flag_variants_mismatch` reports a flag without variants, and the variants have to come back first.
if not any(variant.key for variant in flag.variants):
- return None
- return history
-
-
-def flag_history_skip(ctx: HealthContext) -> CheckSkip | None:
- """The states that `history_to_check` skips, in the order the reader is most likely to act on."""
- skip = flag_setup_skip(ctx, before_launch=False)
- if skip is not None or ctx.flag is None:
- return skip
- if not any(variant.key for variant in ctx.flag.variants):
return not_applicable(NEEDS_VARIANTS)
# Only the health endpoint loads the history, and it does for every state that reaches here, so None there
# means that the load failed.
if ctx.flag_history is None:
return not_checked("Couldn't check because the flag history failed to load")
- return None
+ return ctx.flag_history
-def flag_change_skip(ctx: HealthContext) -> CheckSkip | None:
- skip = flag_history_skip(ctx)
- history = ctx.flag_history
+def changes_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked | CheckSkip:
+ """The flag history that the checks of one kind of change read, or why they don't look."""
+ history = history_to_check(ctx)
# Without the history no change is known. `flag_history_not_readable` reports why.
- if skip is None and isinstance(history, FlagHistoryNotChecked) and history.reason in _HISTORY_NOT_READ:
+ if isinstance(history, FlagHistoryNotChecked) and history.reason in _HISTORY_NOT_READ:
return not_checked(f"Couldn't check because {not_checked_because(history.reason)}")
- return skip
+ return history
-def flag_history_not_readable(ctx: HealthContext) -> ExperimentHealthFinding | None:
+def flag_history_not_readable(ctx: HealthContext) -> CheckOutcome:
"""One finding for every kind of change when the flag's history is not read."""
history = history_to_check(ctx)
+ if isinstance(history, CheckSkip):
+ return history
if not isinstance(history, FlagHistoryNotChecked) or history.reason not in _HISTORY_NOT_READ:
return None
no_access = history.reason == FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS
@@ -135,14 +127,6 @@ def flag_history_not_readable(ctx: HealthContext) -> ExperimentHealthFinding | N
)
-def changes_not_checked[C: FlagChange](history: FlagHistoryNotChecked, kind: type[C]) -> tuple[C, ...]:
- """The changes of one kind that a check reports as not checked. Empty for a history that was not read, which
- `flag_history_not_readable` reports."""
- if history.reason in _HISTORY_NOT_READ:
- return ()
- return history.of_kind(kind)
-
-
def change_finding(
history: FlagHistory,
effect: ChangeEffect,
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index 1b6b022511a5..e36669cafe6d 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -28,7 +28,7 @@ from products.experiments.backend.health.context import (
HealthContext,
PauseHistory,
)
-from products.experiments.backend.health.skips import NO_FLAG, CheckSkip, not_applicable
+from products.experiments.backend.health.skips import NO_FLAG, CheckOutcome, not_applicable
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
describe_split,
@@ -67,14 +67,10 @@ _OUTSIDE_ASSIGNMENT_SETTLED_AFTER = timedelta(days=1)
_NOT_RUNNING_BUT_EXPOSED_TITLE = "The experiment is not running, but users are exposed to multiple variants"
-def flag_state_skip(ctx: HealthContext) -> CheckSkip | None:
- return not_applicable(NO_FLAG) if ctx.flag is None else None
-
-
-def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:
+def flag_state(ctx: HealthContext) -> CheckOutcome:
flag = ctx.flag
if flag is None:
- return None
+ return not_applicable(NO_FLAG)
if flag.deleted:
# A deleted flag distributes no traffic, so the other flag-state findings don't apply. This branch
diff --git a/products/experiments/backend/health/checks/flag_variants.py b/products/experiments/backend/health/checks/flag_variants.py
index b2325d8ba2a4..b6f659b183dc 100644
--- a/products/experiments/backend/health/checks/flag_variants.py
+++ b/products/experiments/backend/health/checks/flag_variants.py
@@ -12,35 +12,31 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import NEEDS_VARIANTS, CheckSkip, flag_setup_skip, not_applicable
+from products.experiments.backend.health.skips import (
+ NEEDS_VARIANTS,
+ CheckOutcome,
+ CheckSkip,
+ flag_to_check,
+ not_applicable,
+)
# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.
FLAG_HAS_NO_VARIANTS = "flag_has_no_variants"
NO_CONTROL_VARIANT = "no_control_variant"
-def flag_variants_skip(ctx: HealthContext) -> CheckSkip | None:
- skip = flag_setup_skip(ctx, before_launch=True)
- if skip is not None or ctx.flag is None:
- return skip
- # A draft without variants is not set up yet, so it has no variants to compare.
- if not ctx.is_running and not any(variant.key for variant in ctx.flag.variants):
- return not_applicable(NEEDS_VARIANTS)
- return None
-
-
-def flag_variants_mismatch(ctx: HealthContext) -> ExperimentHealthFinding | None:
- flag = ctx.flag
+def flag_variants_mismatch(ctx: HealthContext) -> CheckOutcome:
# A deleted flag has a finding of its own, and the flag has to be restored first. After the end the flag
# can change for good reasons, for example to a boolean flag that keeps the shipped variant.
- if flag is None or flag.deleted or ctx.has_ended or ctx.archived:
- return None
+ flag = flag_to_check(ctx, before_launch=True)
+ if isinstance(flag, CheckSkip):
+ return flag
variant_keys = {variant.key for variant in flag.variants if variant.key}
if not variant_keys:
# A draft without variants is not set up yet. A running experiment had at least two at launch.
if not ctx.is_running:
- return None
+ return not_applicable(NEEDS_VARIANTS)
return _finding(
subcode=FLAG_HAS_NO_VARIANTS,
title="The experiment is running, but its feature flag has no variants",
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index f938a1758f2c..dcf024f74044 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -20,7 +20,13 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.flag_state import is_single_variant_shipped, shipped_variant_key
from products.experiments.backend.health.condition_changes import conditions_net_effect
-from products.experiments.backend.health.context import FlagHistory, HealthContext, ReachableCondition, VariantProperty
+from products.experiments.backend.health.context import (
+ FlagHistory,
+ FlagState,
+ HealthContext,
+ ReachableCondition,
+ VariantProperty,
+)
from products.experiments.backend.health.pinned_share import (
PinnedConditions,
PinnedShareGrade,
@@ -29,7 +35,7 @@ from products.experiments.backend.health.pinned_share import (
pinned_share_severity,
pinned_variant_keys,
)
-from products.experiments.backend.health.skips import CheckSkip, flag_setup_skip
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip, flag_to_check
from products.experiments.backend.health.split_changes import format_change_time
from products.feature_flags.backend.facade.edits import ConditionsChange
@@ -56,21 +62,21 @@ class _Pins:
def overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
"""The overrides in place that SOME_CONDITIONS_PINNED is graded by, or None when it is not graded."""
- pins = _pins(ctx)
- if not ctx.is_launched or pins is None or pins.all_pinned:
+ flag = flag_to_check(ctx, before_launch=True)
+ if isinstance(flag, CheckSkip) or not ctx.is_launched:
+ return None
+ pins = _pins(ctx, flag)
+ if pins is None or pins.all_pinned:
return None
return pins.graded
-def forced_variant_skip(ctx: HealthContext) -> CheckSkip | None:
- # The same states as `_pins`, which returns None for them.
- return flag_setup_skip(ctx, before_launch=True)
-
-
-def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFinding | None:
- flag = ctx.flag
- pins = _pins(ctx)
- if flag is None or pins is None:
+def forced_variant_release_condition(ctx: HealthContext) -> CheckOutcome:
+ flag = flag_to_check(ctx, before_launch=True)
+ if isinstance(flag, CheckSkip):
+ return flag
+ pins = _pins(ctx, flag)
+ if pins is None:
return None
is_draft = not ctx.is_launched
@@ -107,11 +113,7 @@ def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFind
)
-def _pins(ctx: HealthContext) -> _Pins | None:
- flag = ctx.flag
- # After the end the flag often holds the shipped decision, not the setup of the run.
- if flag is None or flag.deleted or ctx.has_ended or ctx.archived:
- return None
+def _pins(ctx: HealthContext, flag: FlagState) -> _Pins | None:
pinned = tuple(condition for condition in flag.reachable_conditions if condition.pinned_variant is not None)
if not pinned:
return None
diff --git a/products/experiments/backend/health/checks/metric_event_not_ingested.py b/products/experiments/backend/health/checks/metric_event_not_ingested.py
index 015e50f1626c..ef970e477bd2 100644
--- a/products/experiments/backend/health/checks/metric_event_not_ingested.py
+++ b/products/experiments/backend/health/checks/metric_event_not_ingested.py
@@ -11,13 +11,13 @@ from products.experiments.backend.health.context import HealthContext, MetricEve
from products.experiments.backend.health.skips import (
NOT_AFTER_END,
NOT_ON_ARCHIVED,
- CheckSkip,
+ CheckOutcome,
not_applicable,
not_checked,
)
-def metric_event_not_ingested_skip(ctx: HealthContext) -> CheckSkip | None:
+def metric_event_not_ingested(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
if ctx.has_ended:
@@ -27,12 +27,6 @@ def metric_event_not_ingested_skip(ctx: HealthContext) -> CheckSkip | None:
return not_applicable("Runs once a metric uses an event")
if not names & ctx.events_received.keys():
return not_checked("Couldn't check because PostHog keeps no record of these event names")
- return None
-
-
-def metric_event_not_ingested(ctx: HealthContext) -> ExperimentHealthFinding | None:
- if ctx.archived or ctx.has_ended:
- return None
affected = [
(metric, missing)
for metric in ctx.metric_events
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
index 637c1995c727..34994f8cf937 100644
--- a/products/experiments/backend/health/checks/multiple_variants.py
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -7,7 +7,7 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable, not_checked
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable, not_checked
# Some users get more than one ID in any experiment, for example one per device, so a low share is expected.
# The share grows as users return and log in, so a lower line fires on a large part of all experiments. The cases
@@ -18,19 +18,13 @@ MULTIPLE_VARIANTS_SHARE_THRESHOLD = 5.0 # on the 0-100 scale (5.0 = 5%)
MULTIPLE_VARIANTS_MINIMUM_COUNT = 5
-def multiple_variants_share_skip(ctx: HealthContext) -> CheckSkip | None:
- if ctx.exposures is None:
- return not_applicable(RUNS_AFTER_LAUNCH)
- if ctx.exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
- return not_checked("Couldn't check because the First seen handling doesn't count these users")
- return None
-
-
-def multiple_variants_share(ctx: HealthContext) -> ExperimentHealthFinding | None:
+def multiple_variants_share(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
+ if exposures is None:
+ return not_applicable(RUNS_AFTER_LAUNCH)
# Under "First seen" the exposure query gives each user their first variant, so it counts none of these users.
- if exposures is None or exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
- return None
+ if exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
+ return not_checked("Couldn't check because the First seen handling doesn't count these users")
multiple = exposures.total_exposures.get(MULTIPLE_VARIANT_KEY, 0)
total = sum(exposures.total_exposures.values())
if multiple < MULTIPLE_VARIANTS_MINIMUM_COUNT:
diff --git a/products/experiments/backend/health/checks/new_flag_no_traffic.py b/products/experiments/backend/health/checks/new_flag_no_traffic.py
index 14c2b345047c..984154399fa8 100644
--- a/products/experiments/backend/health/checks/new_flag_no_traffic.py
+++ b/products/experiments/backend/health/checks/new_flag_no_traffic.py
@@ -18,13 +18,13 @@ from products.experiments.backend.health.skips import (
NO_FLAG,
NOT_ON_ARCHIVED,
RUNS_BEFORE_LAUNCH,
- CheckSkip,
+ CheckOutcome,
not_applicable,
not_checked,
)
-def new_flag_no_traffic_skip(ctx: HealthContext) -> CheckSkip | None:
+def new_flag_no_traffic(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
# After launch, the zero exposures check reports an experiment without traffic.
@@ -38,20 +38,14 @@ def new_flag_no_traffic_skip(ctx: HealthContext) -> CheckSkip | None:
return not_checked(
"Couldn't check because this organization's flag calls are stored where this check can't read them"
)
+ if ctx.flag_called:
+ return None
# Without an active flag the web SDK records no call, so the flag gets traffic only after the launch activates it,
# and the finding's advice could not be followed.
- if ctx.flag_called is False and ctx.team_has_active_flag is not True:
+ if ctx.team_has_active_flag is not True:
return not_checked(
"Couldn't check because PostHog's JavaScript SDK only requests flags once a flag in this project is enabled"
)
- return None
-
-
-def new_flag_no_traffic(ctx: HealthContext) -> ExperimentHealthFinding | None:
- if ctx.archived or ctx.is_launched or ctx.flag is None or ctx.flag.deleted:
- return None
- if ctx.flag_called is not False or ctx.team_has_active_flag is not True:
- return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.NEW_FLAG_NO_TRAFFIC,
subcode=None,
diff --git a/products/experiments/backend/health/checks/no_metric.py b/products/experiments/backend/health/checks/no_metric.py
index a6cd7f44c707..b6a3831d625c 100644
--- a/products/experiments/backend/health/checks/no_metric.py
+++ b/products/experiments/backend/health/checks/no_metric.py
@@ -5,16 +5,14 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
-def no_metric_skip(ctx: HealthContext) -> CheckSkip | None:
- return None if ctx.is_launched else not_applicable(RUNS_AFTER_LAUNCH)
-
-
-def no_metric(ctx: HealthContext) -> ExperimentHealthFinding | None:
+def no_metric(ctx: HealthContext) -> CheckOutcome:
+ if not ctx.is_launched:
+ return not_applicable(RUNS_AFTER_LAUNCH)
# The deprecated `filters` metric does not count, because no engine reads it.
- if not ctx.is_launched or ctx.primary_metric_count + ctx.secondary_metric_count > 0:
+ if ctx.primary_metric_count + ctx.secondary_metric_count > 0:
return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.NO_METRIC,
diff --git a/products/experiments/backend/health/checks/split_changed.py b/products/experiments/backend/health/checks/split_changed.py
index e6302d7c418e..c797e65a88b0 100644
--- a/products/experiments/backend/health/checks/split_changed.py
+++ b/products/experiments/backend/health/checks/split_changed.py
@@ -18,16 +18,16 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.change_wording import duration, suggested_start, the_change
from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
- changes_not_checked,
+ changes_to_check,
delay_after_launch,
exposed_when,
few_exposed_units,
- history_to_check,
leave_out,
not_checked_finding,
)
from products.experiments.backend.health.checks.flag_state import is_single_variant_shipped
from products.experiments.backend.health.context import FlagHistory, FlagHistoryNotChecked, FlagState, HealthContext
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
SplitNetEffect,
@@ -48,10 +48,13 @@ _REVERTED_INFO_TITLE = "The variant split changed and was set back"
_NOT_CHECKED_TITLE = "Variant split changes were not checked"
-def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
- history = history_to_check(ctx)
+def split_changed_mid_run(ctx: HealthContext) -> CheckOutcome:
+ history = changes_to_check(ctx)
+ if isinstance(history, CheckSkip):
+ return history
flag = ctx.flag
- if history is None or flag is None:
+ # `changes_to_check` skips an experiment without a flag, but the type doesn't carry that.
+ if flag is None:
return None
if isinstance(history, FlagHistoryNotChecked):
return _not_checked(history, flag)
@@ -105,7 +108,7 @@ def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
def _not_checked(history: FlagHistoryNotChecked, flag: FlagState) -> ExperimentHealthFinding | None:
- effect = net_effect(changes_not_checked(history, SplitChange))
+ effect = net_effect(history.of_kind(SplitChange))
if effect is None or _reported_as_shipped(effect, flag):
return None
at = format_change_time(effect.last.changed_at, history.team_timezone)
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index b760ab36a4ba..91646d197203 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -4,15 +4,16 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
SAMPLE_RATIO_MISMATCH_P_VALUE = 0.001
-def srm_skip(ctx: HealthContext) -> CheckSkip | None:
+def srm(ctx: HealthContext) -> CheckOutcome:
if ctx.exposures is None:
return not_applicable(RUNS_AFTER_LAUNCH)
- if ctx.exposures.sample_ratio_mismatch_p_value is None:
+ p_value = ctx.exposures.sample_ratio_mismatch_p_value
+ if p_value is None:
# The exposure query compares only the variants with a share of the split that the analysis keeps, so after
# a variant ships there is nothing to compare however many users are exposed.
variants_in_split = [
@@ -23,12 +24,7 @@ def srm_skip(ctx: HealthContext) -> CheckSkip | None:
if len(variants_in_split) < 2:
return not_applicable("Needs two or more variants in the split")
return not_applicable("Runs once enough users are exposed")
- return None
-
-
-def srm(ctx: HealthContext) -> ExperimentHealthFinding | None:
- p_value = ctx.exposures.sample_ratio_mismatch_p_value if ctx.exposures else None
- if p_value is None or p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
+ if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.SRM,
diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.py
index de6d1de028d8..4ccf7ffe7510 100644
--- a/products/experiments/backend/health/checks/zero_exposures.py
+++ b/products/experiments/backend/health/checks/zero_exposures.py
@@ -7,7 +7,7 @@ from products.experiments.backend.facade.contracts import (
from products.experiments.backend.health.checks.exposure_criteria import exposure_criteria_cannot_match
from products.experiments.backend.health.context import HealthContext
from products.experiments.backend.health.delays import LAST_CALLED_AT_LAG, ZERO_EXPOSURES_GRACE_HOURS
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.
EXPERIMENT_PAUSED = "experiment_paused"
@@ -16,26 +16,17 @@ CALLS_NOT_COUNTED = "calls_not_counted"
CUSTOM_EXPOSURE_NOT_MATCHED = "custom_exposure_not_matched"
-def zero_exposures_skip(ctx: HealthContext) -> CheckSkip | None:
+def zero_exposures(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None or exposures.hours_since_launch is None or not ctx.is_launched:
return not_applicable(RUNS_AFTER_LAUNCH)
if exposures.hours_since_launch < ZERO_EXPOSURES_GRACE_HOURS:
return not_applicable(f"Runs {ZERO_EXPOSURES_GRACE_HOURS} hours after launch")
- if exposure_criteria_cannot_match(ctx) is not None:
- return not_applicable("The exposure criteria check reports why no users are exposed")
- return None
-
-
-def zero_exposures(ctx: HealthContext) -> ExperimentHealthFinding | None:
- exposures = ctx.exposures
- if exposures is None or exposures.hours_since_launch is None or not ctx.is_launched:
- return None
- if exposures.hours_since_launch < ZERO_EXPOSURES_GRACE_HOURS or sum(exposures.total_exposures.values()) > 0:
- return None
# Criteria that no event can match leave nobody exposed, and the exposure criteria check names that cause. This
# check calls it rather than repeating its rules, so exactly one of the two reports the experiment.
- if exposure_criteria_cannot_match(ctx) is not None:
+ if isinstance(exposure_criteria_cannot_match(ctx), ExperimentHealthFinding):
+ return not_applicable("The exposure criteria check reports why no users are exposed")
+ if sum(exposures.total_exposures.values()) > 0:
return None
evidence = {"hours_since_launch": int(exposures.hours_since_launch)}
# A group experiment counts groups, by the rule the exposure query uses.
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index dd79db50ef57..1b1411713947 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -1,4 +1,5 @@
from collections.abc import Callable, Sequence
+from typing import assert_never
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
@@ -10,124 +11,93 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckStatus,
ExperimentHealthFinding,
)
-from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded, bias_risk_skip
+from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
-from products.experiments.backend.health.checks.early_significance import early_significance, early_significance_skip
-from products.experiments.backend.health.checks.exposure_criteria import (
- exposure_criteria_cannot_match,
- exposure_criteria_skip,
-)
-from products.experiments.backend.health.checks.flag_change_findings import (
- flag_change_skip,
- flag_history_not_readable,
- flag_history_skip,
-)
-from products.experiments.backend.health.checks.flag_state import flag_state, flag_state_skip
-from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch, flag_variants_skip
-from products.experiments.backend.health.checks.forced_variant import (
- forced_variant_release_condition,
- forced_variant_skip,
-)
-from products.experiments.backend.health.checks.metric_event_not_ingested import (
- metric_event_not_ingested,
- metric_event_not_ingested_skip,
-)
-from products.experiments.backend.health.checks.multiple_variants import (
- multiple_variants_share,
- multiple_variants_share_skip,
-)
-from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic, new_flag_no_traffic_skip
-from products.experiments.backend.health.checks.no_metric import no_metric, no_metric_skip
+from products.experiments.backend.health.checks.early_significance import early_significance
+from products.experiments.backend.health.checks.exposure_criteria import exposure_criteria_cannot_match
+from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
+from products.experiments.backend.health.checks.flag_state import flag_state
+from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
+from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
+from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested
+from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
+from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic
+from products.experiments.backend.health.checks.no_metric import no_metric
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.checks.srm import srm, srm_skip
-from products.experiments.backend.health.checks.zero_exposures import zero_exposures, zero_exposures_skip
+from products.experiments.backend.health.checks.srm import srm
+from products.experiments.backend.health.checks.zero_exposures import zero_exposures
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import CheckSkip
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
@frozen
class HealthCheck:
key: ExperimentHealthCheckKey
name: str
- # Returns None when it finds no problem, and also when the context lacks an input it reads.
- find: Callable[[HealthContext], ExperimentHealthFinding | None]
- # Why the check does not look, or None when it looks. Keep in step with the guards of `find`.
- skip: Callable[[HealthContext], CheckSkip | None]
+ run: Callable[[HealthContext], CheckOutcome]
EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_STATE,
name="Feature flag rollout",
- find=flag_state,
- skip=flag_state_skip,
+ run=flag_state,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_HISTORY,
name="Feature flag history",
- find=flag_history_not_readable,
- skip=flag_history_skip,
+ run=flag_history_not_readable,
),
HealthCheck(
key=ExperimentHealthCheckKey.SPLIT_CHANGED_MID_RUN,
name="Variant split changes since launch",
- find=split_changed_mid_run,
- skip=flag_change_skip,
+ run=split_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.BUCKETING_CHANGED_MID_RUN,
name="Flag key and bucketing changes since launch",
- find=bucketing_changed_mid_run,
- skip=flag_change_skip,
+ run=bucketing_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.RELEASE_CONDITIONS_CHANGED_MID_RUN,
name="Release condition changes since launch",
- find=release_conditions_changed_mid_run,
- skip=flag_change_skip,
+ run=release_conditions_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_VARIANTS_MISMATCH,
name="Feature flag variants",
- find=flag_variants_mismatch,
- skip=flag_variants_skip,
+ run=flag_variants_mismatch,
),
HealthCheck(
key=ExperimentHealthCheckKey.FORCED_VARIANT_RELEASE_CONDITION,
name="Release conditions that force a variant",
- find=forced_variant_release_condition,
- skip=forced_variant_skip,
+ run=forced_variant_release_condition,
),
HealthCheck(
key=ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC,
name="Feature flag traffic",
- find=new_flag_no_traffic,
- skip=new_flag_no_traffic_skip,
+ run=new_flag_no_traffic,
),
HealthCheck(
key=ExperimentHealthCheckKey.NO_METRIC,
name="Metrics",
- find=no_metric,
- skip=no_metric_skip,
+ run=no_metric,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_EVENT_NOT_INGESTED,
name="Metric events",
- find=metric_event_not_ingested,
- skip=metric_event_not_ingested_skip,
+ run=metric_event_not_ingested,
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,
name="Exposure criteria",
- find=exposure_criteria_cannot_match,
- skip=exposure_criteria_skip,
+ run=exposure_criteria_cannot_match,
),
HealthCheck(
key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,
name="Sample size behind significant results",
- find=early_significance,
- skip=early_significance_skip,
+ run=early_significance,
),
)
@@ -137,26 +107,22 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.ZERO_EXPOSURES,
name="Exposures",
- find=zero_exposures,
- skip=zero_exposures_skip,
+ run=zero_exposures,
),
HealthCheck(
key=ExperimentHealthCheckKey.SRM,
name="Split of users across variants",
- find=srm,
- skip=srm_skip,
+ run=srm,
),
HealthCheck(
key=ExperimentHealthCheckKey.BIAS_RISK_MULTIPLE_EXCLUDED,
name="Bias from an uneven split",
- find=bias_risk_multiple_excluded,
- skip=bias_risk_skip,
+ run=bias_risk_multiple_excluded,
),
HealthCheck(
key=ExperimentHealthCheckKey.MULTIPLE_VARIANTS_SHARE,
name="Users exposed to multiple variants",
- find=multiple_variants_share,
- skip=multiple_variants_share_skip,
+ run=multiple_variants_share,
),
)
@@ -169,23 +135,23 @@ def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> ExperimentHea
results: list[ExperimentHealthCheckResult] = []
for check in checks:
try:
- # Every check runs whatever its skip says, so a skip that is out of step with the check's guards can't
- # hide a finding. The skip is read only without a finding, so a failing skip can't hide one either.
- finding = check.find(ctx)
- skip = check.skip(ctx) if finding is None else None
+ outcome = check.run(ctx)
except Exception as error:
# The findings ride on the health action, the exposure answer and `experiment launched`, so one
# failing check must not fail them or hide the findings of the other checks.
capture_exception(error)
results.append(_result(check, ExperimentHealthCheckStatus.NOT_CHECKED, _CHECK_FAILED))
continue
- if finding is not None:
- findings.append(finding)
- results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None))
- elif skip is not None:
- results.append(_result(check, skip.status, skip.reason))
- else:
- results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None))
+ match outcome:
+ case CheckSkip(status=status, reason=reason):
+ results.append(_result(check, status, reason))
+ case ExperimentHealthFinding():
+ findings.append(outcome)
+ results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None))
+ case None:
+ results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None))
+ case _:
+ assert_never(outcome)
return ExperimentHealth(findings=tuple(findings), checks=tuple(results))
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
index 3b765dcb5c0d..305033c08bca 100644
--- a/products/experiments/backend/health/skips.py
+++ b/products/experiments/backend/health/skips.py
@@ -2,8 +2,8 @@ from typing import Literal
from posthog.dataclasses import frozen
-from products.experiments.backend.facade.contracts import ExperimentHealthCheckStatus
-from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.facade.contracts import ExperimentHealthCheckStatus, ExperimentHealthFinding
+from products.experiments.backend.health.context import FlagState, HealthContext
@frozen
@@ -12,6 +12,9 @@ class CheckSkip:
reason: str
+# What a check returns: a finding, why the check doesn't look, or None when it looks and finds no problem.
+CheckOutcome = ExperimentHealthFinding | CheckSkip | None
+
RUNS_AFTER_LAUNCH = "Runs after launch"
RUNS_BEFORE_LAUNCH = "Runs only before launch"
NOT_AFTER_END = "Doesn't run after the experiment ends"
@@ -29,17 +32,18 @@ def not_checked(reason: str) -> CheckSkip:
return CheckSkip(status=ExperimentHealthCheckStatus.NOT_CHECKED, reason=reason)
-def flag_setup_skip(ctx: HealthContext, *, before_launch: bool) -> CheckSkip | None:
- """The skip of a check of the flag's setup. After the end the flag often holds the shipped decision, not the
- setup of the run, so these checks stop there."""
+def flag_to_check(ctx: HealthContext, *, before_launch: bool) -> FlagState | CheckSkip:
+ """The flag that a check of the flag's setup reads, or why the check doesn't look. After the end the flag often
+ holds the shipped decision, not the setup of the run, so these checks stop there."""
+ flag = ctx.flag
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
- if ctx.flag is None:
+ if flag is None:
return not_applicable(NO_FLAG)
- if ctx.flag.deleted:
+ if flag.deleted:
return not_applicable(FLAG_DELETED)
if ctx.has_ended:
return not_applicable(NOT_AFTER_END)
if not before_launch and not ctx.is_launched:
return not_applicable(RUNS_AFTER_LAUNCH)
- return None
+ return flag
diff --git a/products/experiments/backend/health/test/test_bucketing_changed.py b/products/experiments/backend/health/test/test_bucketing_changed.py
index e654f49cdcd5..49b9222f64a7 100644
--- a/products/experiments/backend/health/test/test_bucketing_changed.py
+++ b/products/experiments/backend/health/test/test_bucketing_changed.py
@@ -7,6 +7,7 @@ from unittest import TestCase
from parameterized import parameterized
from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
@@ -14,6 +15,13 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
from products.experiments.backend.health.context import FlagHistoryNotCheckedReason, HealthContext, parse_flag_state
+from products.experiments.backend.health.skips import (
+ NEEDS_VARIANTS,
+ NOT_AFTER_END,
+ CheckSkip,
+ not_applicable,
+ not_checked,
+)
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT, flag_history
from products.feature_flags.backend.facade.edits import BucketingChange, BucketingSetting
@@ -215,7 +223,7 @@ class TestBucketingChangedMidRun(TestCase):
) -> None:
finding = bucketing_changed_mid_run(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(finding.code, ExperimentHealthFindingCode.BUCKETING_CHANGED_MID_RUN)
self.assertEqual((finding.subcode, finding.severity, finding.title), expected)
self.assertEqual(finding.detail, expected_detail)
@@ -223,12 +231,20 @@ class TestBucketingChangedMidRun(TestCase):
@parameterized.expand(
[
- ("no_change", _context()),
- ("ended", _context((KEY, "old-key", "new-key"), is_running=False)),
- ("flag_without_variants", _context((KEY, "old-key", "new-key"), variants=[])),
+ ("no_change", _context(), None),
+ ("ended", _context((KEY, "old-key", "new-key"), is_running=False), not_applicable(NOT_AFTER_END)),
+ (
+ "flag_without_variants",
+ _context((KEY, "old-key", "new-key"), variants=[]),
+ not_applicable(NEEDS_VARIANTS),
+ ),
# The split check reports a history the reader can't view.
- ("no_history_access", _context(not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS)),
+ (
+ "no_history_access",
+ _context(not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),
+ not_checked("Couldn't check because you can't view the linked feature flag's history"),
+ ),
]
)
- def test_reports_nothing(self, _name: str, ctx: HealthContext) -> None:
- self.assertIsNone(bucketing_changed_mid_run(ctx))
+ def test_reports_nothing(self, _name: str, ctx: HealthContext, expected: CheckSkip | None) -> None:
+ self.assertEqual(bucketing_changed_mid_run(ctx), expected)
diff --git a/products/experiments/backend/health/test/test_conditions_changed.py b/products/experiments/backend/health/test/test_conditions_changed.py
index 82a9f0ea1db5..59b97029de24 100644
--- a/products/experiments/backend/health/test/test_conditions_changed.py
+++ b/products/experiments/backend/health/test/test_conditions_changed.py
@@ -7,6 +7,7 @@ from unittest import TestCase
from parameterized import parameterized
from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
ExperimentHealthFlagChangeKind,
@@ -19,6 +20,7 @@ from products.experiments.backend.health.context import (
HealthContext,
parse_flag_state,
)
+from products.experiments.backend.health.skips import CheckSkip, not_checked
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT, flag_history
from products.feature_flags.backend.facade.edits import ConditionsChange, parse_release_conditions
@@ -367,7 +369,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):
) -> None:
finding = release_conditions_changed_mid_run(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(finding.code, ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN)
self.assertEqual((finding.subcode, finding.severity, finding.title, finding.diagnostic_ref), expected)
if expected_detail is not None:
@@ -375,21 +377,27 @@ class TestReleaseConditionsChangedMidRun(TestCase):
@parameterized.expand(
[
- ("no_change", _context([PRO])),
- ("set_back_with_nobody_in_between", _context([PRO], [EVERYONE], [PRO], exposed_between=False)),
- ("no_history_access", _context([PRO], not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS)),
+ ("no_change", _context([PRO]), None),
+ ("set_back_with_nobody_in_between", _context([PRO], [EVERYONE], [PRO], exposed_between=False), None),
+ (
+ "no_history_access",
+ _context([PRO], not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),
+ not_checked("Couldn't check because you can't view the linked feature flag's history"),
+ ),
# The forced-variant check reports an override in place, with the time it was set.
- ("override_added", _context([PRO, EVERYONE], [PRO_PINNED, EVERYONE])),
- ("pinned_condition_added", _context([EVERYONE], [PRO_PINNED, EVERYONE])),
+ ("override_added", _context([PRO, EVERYONE], [PRO_PINNED, EVERYONE]), None),
+ ("pinned_condition_added", _context([EVERYONE], [PRO_PINNED, EVERYONE]), None),
(
"override_changed_to_another_variant",
_context([PRO_PINNED, EVERYONE], [{**PRO, "variant": "control"}, EVERYONE]),
+ None,
),
(
"override_removed_from_under_one_percent",
_context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 10, 2: 2990}),
+ None,
),
]
)
- def test_reports_nothing(self, _name: str, ctx: HealthContext) -> None:
- self.assertIsNone(release_conditions_changed_mid_run(ctx))
+ def test_reports_nothing(self, _name: str, ctx: HealthContext, expected: CheckSkip | None) -> None:
+ self.assertEqual(release_conditions_changed_mid_run(ctx), expected)
diff --git a/products/experiments/backend/health/test/test_early_significance.py b/products/experiments/backend/health/test/test_early_significance.py
index f1c42fc8f753..b37759a019e1 100644
--- a/products/experiments/backend/health/test/test_early_significance.py
+++ b/products/experiments/backend/health/test/test_early_significance.py
@@ -13,16 +13,14 @@ from products.experiments.backend.facade.api import get_experiment_health
from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckKey,
ExperimentHealthCheckStatus,
+ ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
ExperimentHealthReaderAccess,
)
-from products.experiments.backend.health.checks.early_significance import (
- BELOW_DETECTABLE_EFFECT,
- early_significance,
- early_significance_skip,
-)
+from products.experiments.backend.health.checks.early_significance import BELOW_DETECTABLE_EFFECT, early_significance
from products.experiments.backend.health.context import HealthContext, PrimaryMetric
+from products.experiments.backend.health.skips import CheckSkip
from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
from products.experiments.backend.logic.result_stats import BaselineStats, Interval, MetricResultStats, VariantStats
from products.experiments.backend.metric_calculation.config import build_primary_calculation_configs
@@ -107,12 +105,14 @@ class TestEarlySignificance(TestCase):
) -> None:
finding = early_significance(ctx)
- self.assertEqual(finding.evidence["variant"] if finding else None, expected_variant)
+ self.assertEqual(
+ finding.evidence["variant"] if isinstance(finding, ExperimentHealthFinding) else finding, expected_variant
+ )
def test_states_the_analysis_difference_and_the_planned_sizes(self) -> None:
finding = early_significance(SMALL_SIGNIFICANT_LIFT)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.code, finding.subcode, finding.severity),
(
@@ -165,10 +165,9 @@ class TestEarlySignificance(TestCase):
]
)
def test_says_why_it_did_not_look(self, _name: str, ctx: HealthContext, expected: Status) -> None:
- skip = early_significance_skip(ctx)
+ outcome = early_significance(ctx)
- self.assertEqual(skip.status if skip else None, expected)
- self.assertIsNone(early_significance(ctx))
+ self.assertEqual(outcome.status if isinstance(outcome, CheckSkip) else outcome, expected)
class TestEarlySignificanceFromStoredResults(BaseTest):
diff --git a/products/experiments/backend/health/test/test_exposure_criteria.py b/products/experiments/backend/health/test/test_exposure_criteria.py
index d5cf19100673..669b289c0b04 100644
--- a/products/experiments/backend/health/test/test_exposure_criteria.py
+++ b/products/experiments/backend/health/test/test_exposure_criteria.py
@@ -7,13 +7,20 @@ from parameterized import parameterized
from posthog.schema import PropertyOperator
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverity
from products.experiments.backend.health.checks.exposure_criteria import (
EXPOSURE_EVENT_NOT_INGESTED,
FILTER_ON_ANOTHER_FLAG,
exposure_criteria_cannot_match,
)
from products.experiments.backend.health.context import ExposureEvents, FlagKeyFilter, HealthContext
+from products.experiments.backend.health.skips import (
+ NOT_AFTER_END,
+ NOT_ON_ARCHIVED,
+ CheckSkip,
+ not_applicable,
+ not_checked,
+)
from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING_CONTEXT
Severity = ExperimentHealthFindingSeverity
@@ -59,7 +66,7 @@ class TestExposureCriteriaCannotMatch(TestCase):
("no_values_match_every_event", _flag_filter(), None),
("a_number_can_be_converted", _flag_filter(7), None),
("not_exact", _flag_filter("pricing-test", operator=PropertyOperator.IS_NOT), None),
- ("archived", _flag_filter("pricing-test", archived=True), None),
+ ("archived", _flag_filter("pricing-test", archived=True), not_applicable(NOT_ON_ARCHIVED)),
(
"custom_event_in_a_draft",
_custom_event({"checkout_viewed": False}, **DRAFT, hours_since_launch=None),
@@ -76,13 +83,22 @@ class TestExposureCriteriaCannotMatch(TestCase):
(EXPOSURE_EVENT_NOT_INGESTED, Severity.WARNING),
),
("custom_event_received", _custom_event({"checkout_viewed": True}), None),
- ("custom_event_without_a_definition", _custom_event({}), None),
- ("custom_event_after_the_end", _custom_event({"checkout_viewed": False}, **ENDED), None),
+ (
+ "custom_event_without_a_definition",
+ _custom_event({}),
+ not_checked("Couldn't check because PostHog keeps no record of this event name"),
+ ),
+ (
+ "custom_event_after_the_end",
+ _custom_event({"checkout_viewed": False}, **ENDED),
+ not_applicable(NOT_AFTER_END),
+ ),
]
)
def test_exposure_criteria_cannot_match(
- self, _name: str, ctx: HealthContext, expected: tuple[str, Severity] | None
+ self, _name: str, ctx: HealthContext, expected: tuple[str, Severity] | CheckSkip | None
) -> None:
- finding = exposure_criteria_cannot_match(ctx)
+ outcome = exposure_criteria_cannot_match(ctx)
- self.assertEqual((finding.subcode, finding.severity) if finding else None, expected)
+ actual = (outcome.subcode, outcome.severity) if isinstance(outcome, ExperimentHealthFinding) else outcome
+ self.assertEqual(actual, expected)
diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.py
index 3879f59261e3..bb433a411f38 100644
--- a/products/experiments/backend/health/test/test_flag_state.py
+++ b/products/experiments/backend/health/test/test_flag_state.py
@@ -6,7 +6,11 @@ from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode, ExperimentHealthFindingSeverity
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+)
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.context import (
ConditionSetTraffic,
@@ -200,7 +204,7 @@ class TestFlagState(TestCase):
if expected is None:
self.assertIsNone(finding)
return
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
code, subcode, variant_key = expected
self.assertEqual((finding.code, finding.subcode), (code, subcode))
self.assertEqual(finding.evidence.get("variant_key"), variant_key)
@@ -262,7 +266,7 @@ class TestFlagState(TestCase):
) -> None:
finding = flag_state(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual((finding.subcode, finding.severity, finding.title), expected)
def test_a_pause_from_the_experiment_names_its_date(self) -> None:
@@ -275,7 +279,7 @@ class TestFlagState(TestCase):
)
)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.detail, dict(finding.evidence)),
(
@@ -290,7 +294,7 @@ class TestFlagState(TestCase):
def test_variants_assigned_outside_the_flag_service_are_info(self) -> None:
finding = flag_state(_context(RUNNING, active=True, filters=ZERO_ROLLOUT, traffic=_traffic(1234)))
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.severity, finding.title, finding.detail, dict(finding.evidence)),
(
diff --git a/products/experiments/backend/health/test/test_flag_variants.py b/products/experiments/backend/health/test/test_flag_variants.py
index cb4b314373b1..ca00963d06da 100644
--- a/products/experiments/backend/health/test/test_flag_variants.py
+++ b/products/experiments/backend/health/test/test_flag_variants.py
@@ -5,9 +5,17 @@ from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingCode
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
from products.experiments.backend.health.context import HealthContext, parse_flag_state
+from products.experiments.backend.health.skips import (
+ FLAG_DELETED,
+ NEEDS_VARIANTS,
+ NOT_AFTER_END,
+ NOT_ON_ARCHIVED,
+ CheckSkip,
+ not_applicable,
+)
from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status
EVEN_SPLIT = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
@@ -40,24 +48,31 @@ class TestFlagVariantsMismatch(TestCase):
@parameterized.expand(
[
("running_flag_without_variants", _context(RUNNING, variants=None), "flag_has_no_variants"),
- ("draft_flag_without_variants", _context(DRAFT, variants=None), None),
+ ("draft_flag_without_variants", _context(DRAFT, variants=None), not_applicable(NEEDS_VARIANTS)),
("running_baseline_not_a_variant", _context(RUNNING, baseline="holdout"), "no_control_variant"),
("draft_baseline_not_a_variant", _context(DRAFT, baseline="holdout"), "no_control_variant"),
("baseline_is_a_variant", _context(RUNNING, baseline="test"), None),
("no_baseline_and_no_control_variant", _context(RUNNING, variants=NO_CONTROL_SPLIT), None),
- ("ended_flag_changed_to_boolean", _context(ENDED, variants=None), None),
- ("archived_draft_baseline_not_a_variant", _context(DRAFT, baseline="holdout", archived=True), None),
- ("deleted_flag_without_variants", _context(RUNNING, variants=None, deleted=True), None),
+ ("ended_flag_changed_to_boolean", _context(ENDED, variants=None), not_applicable(NOT_AFTER_END)),
+ (
+ "archived_draft_baseline_not_a_variant",
+ _context(DRAFT, baseline="holdout", archived=True),
+ not_applicable(NOT_ON_ARCHIVED),
+ ),
+ (
+ "deleted_flag_without_variants",
+ _context(RUNNING, variants=None, deleted=True),
+ not_applicable(FLAG_DELETED),
+ ),
]
)
- def test_flag_variants_mismatch(self, _name: str, ctx: HealthContext, expected_subcode: str | None) -> None:
- finding = flag_variants_mismatch(ctx)
+ def test_flag_variants_mismatch(self, _name: str, ctx: HealthContext, expected: str | CheckSkip | None) -> None:
+ outcome = flag_variants_mismatch(ctx)
- if expected_subcode is None:
- self.assertIsNone(finding)
+ if not isinstance(outcome, ExperimentHealthFinding) or not isinstance(expected, str):
+ self.assertEqual(outcome, expected)
return
- assert finding is not None
self.assertEqual(
- (finding.code, finding.subcode), (ExperimentHealthFindingCode.FLAG_VARIANTS_MISMATCH, expected_subcode)
+ (outcome.code, outcome.subcode), (ExperimentHealthFindingCode.FLAG_VARIANTS_MISMATCH, expected)
)
- self.assertEqual(finding.evidence.get("baseline_variant_key"), ctx.baseline_variant_key)
+ self.assertEqual(outcome.evidence.get("baseline_variant_key"), ctx.baseline_variant_key)
diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.py
index 972857ca5271..dc92653ce0ef 100644
--- a/products/experiments/backend/health/test/test_forced_variant.py
+++ b/products/experiments/backend/health/test/test_forced_variant.py
@@ -6,7 +6,7 @@ from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverity
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
from products.experiments.backend.health.condition_set_traffic import grades_by_pinned_share
from products.experiments.backend.health.context import (
@@ -16,6 +16,7 @@ from products.experiments.backend.health.context import (
HealthContext,
parse_flag_state,
)
+from products.experiments.backend.health.skips import FLAG_DELETED, NOT_AFTER_END, CheckSkip, not_applicable
from products.experiments.backend.health.test.factories import (
DRAFT,
ENDED,
@@ -192,8 +193,12 @@ class TestForcedVariantReleaseCondition(TestCase):
_context([{**EVERYONE, "variant": "control"}], variants=SHIPPED_SPLIT),
("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),
),
- ("ended", _context([{**EVERYONE, "variant": "control"}], status=ENDED), None),
- ("deleted_flag", _context([{**EVERYONE, "variant": "control"}], deleted=True), None),
+ ("ended", _context([{**EVERYONE, "variant": "control"}], status=ENDED), not_applicable(NOT_AFTER_END)),
+ (
+ "deleted_flag",
+ _context([{**EVERYONE, "variant": "control"}], deleted=True),
+ not_applicable(FLAG_DELETED),
+ ),
(
"pinned_share_under_one_percent",
_context([{**STAFF, "variant": "test"}, EVERYONE], traffic=_traffic({1: 5, 2: 995})),
@@ -290,14 +295,13 @@ class TestForcedVariantReleaseCondition(TestCase):
self,
_name: str,
ctx: HealthContext,
- expected: tuple[str, ExperimentHealthFindingSeverity, str, str, float | None] | None,
+ expected: tuple[str, ExperimentHealthFindingSeverity, str, str, float | None] | CheckSkip | None,
) -> None:
finding = forced_variant_release_condition(ctx)
- if expected is None:
- self.assertIsNone(finding)
+ if not isinstance(finding, ExperimentHealthFinding) or not isinstance(expected, tuple):
+ self.assertEqual(finding, expected)
return
- assert finding is not None
self.assertEqual(
(
finding.subcode,
@@ -344,7 +348,7 @@ class TestForcedVariantReleaseCondition(TestCase):
) -> None:
finding = forced_variant_release_condition(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.evidence.get("reason"), "can't measure" in finding.detail), (expected_reason, says_not_measurable)
)
@@ -384,7 +388,7 @@ class TestForcedVariantReleaseCondition(TestCase):
_context([{**STAFF, "variant": "test"}, EVERYONE], groups_at_launch=groups_at_launch)
)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
"The override of Set 1 was set on September 12, 2026 at 3:04 PM CEST, after launch, so users it matches "
"who were exposed before may have switched variants." in finding.detail,
@@ -408,7 +412,7 @@ class TestForcedVariantReleaseCondition(TestCase):
)
)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.title, finding.detail, finding.evidence["variant_property"]),
(
diff --git a/products/experiments/backend/health/test/test_metric_event_not_ingested.py b/products/experiments/backend/health/test/test_metric_event_not_ingested.py
index 8337d6fefdb0..cd786a823952 100644
--- a/products/experiments/backend/health/test/test_metric_event_not_ingested.py
+++ b/products/experiments/backend/health/test/test_metric_event_not_ingested.py
@@ -5,9 +5,16 @@ from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingSeverity
from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested
from products.experiments.backend.health.context import HealthContext, MetricEvents
+from products.experiments.backend.health.skips import (
+ NOT_AFTER_END,
+ NOT_ON_ARCHIVED,
+ CheckSkip,
+ not_applicable,
+ not_checked,
+)
from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING_CONTEXT
Severity = ExperimentHealthFindingSeverity
@@ -35,15 +42,25 @@ class TestMetricEventNotIngested(TestCase):
),
("only_a_secondary_metric", _context(SIGNUPS, received={"signup": False}), Severity.INFO),
("received", _context(REVENUE, received={"purchase": True}), None),
- ("a_name_without_a_definition_is_left_out", _context(REVENUE, received={}), None),
- ("ended", _context(REVENUE, received={"purchase": False}, **ENDED), None),
- ("archived", _context(REVENUE, received={"purchase": False}, archived=True), None),
+ (
+ "a_name_without_a_definition_is_left_out",
+ _context(REVENUE, received={}),
+ not_checked("Couldn't check because PostHog keeps no record of these event names"),
+ ),
+ ("ended", _context(REVENUE, received={"purchase": False}, **ENDED), not_applicable(NOT_AFTER_END)),
+ (
+ "archived",
+ _context(REVENUE, received={"purchase": False}, archived=True),
+ not_applicable(NOT_ON_ARCHIVED),
+ ),
]
)
- def test_severity_follows_the_state(self, _name: str, ctx: HealthContext, expected: Severity | None) -> None:
- finding = metric_event_not_ingested(ctx)
+ def test_severity_follows_the_state(
+ self, _name: str, ctx: HealthContext, expected: Severity | CheckSkip | None
+ ) -> None:
+ outcome = metric_event_not_ingested(ctx)
- self.assertEqual(finding.severity if finding else None, expected)
+ self.assertEqual(outcome.severity if isinstance(outcome, ExperimentHealthFinding) else outcome, expected)
def test_names_each_metric_and_its_missing_events(self) -> None:
funnel = MetricEvents(name="Checkout", role="primary", events=("cart", "pay", "view"))
@@ -51,7 +68,7 @@ class TestMetricEventNotIngested(TestCase):
finding = metric_event_not_ingested(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(finding.title, "2 metrics use events this project hasn't received")
self.assertEqual(
dict(finding.evidence),
diff --git a/products/experiments/backend/health/test/test_no_metric.py b/products/experiments/backend/health/test/test_no_metric.py
index 0977d2bc473c..fad3b5bc77c6 100644
--- a/products/experiments/backend/health/test/test_no_metric.py
+++ b/products/experiments/backend/health/test/test_no_metric.py
@@ -5,11 +5,14 @@ from unittest import TestCase
from parameterized import parameterized
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingCode
from products.experiments.backend.health.checks.no_metric import no_metric
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
+NO_METRIC = ExperimentHealthFindingCode.NO_METRIC
+
def _context(**changes: Any) -> HealthContext:
return replace(RUNNING_CONTEXT, **{"primary_metric_count": 0, **changes})
@@ -18,14 +21,20 @@ def _context(**changes: Any) -> HealthContext:
class TestNoMetric(TestCase):
@parameterized.expand(
[
- ("running_without_metrics", _context(), True),
- ("ended_without_metrics", _context(is_running=False, has_ended=True), True),
- ("draft_without_metrics", _context(is_launched=False, is_running=False), False),
- ("secondary_metric_only", _context(secondary_metric_count=1), False),
- ("primary_metric_only", _context(primary_metric_count=1), False),
+ ("running_without_metrics", _context(), NO_METRIC),
+ ("ended_without_metrics", _context(is_running=False, has_ended=True), NO_METRIC),
+ (
+ "draft_without_metrics",
+ _context(is_launched=False, is_running=False),
+ not_applicable(RUNS_AFTER_LAUNCH),
+ ),
+ ("secondary_metric_only", _context(secondary_metric_count=1), None),
+ ("primary_metric_only", _context(primary_metric_count=1), None),
]
)
- def test_no_metric(self, _name: str, ctx: HealthContext, expected: bool) -> None:
- finding = no_metric(ctx)
+ def test_no_metric(
+ self, _name: str, ctx: HealthContext, expected: ExperimentHealthFindingCode | CheckSkip | None
+ ) -> None:
+ outcome = no_metric(ctx)
- self.assertEqual(finding.code if finding else None, ExperimentHealthFindingCode.NO_METRIC if expected else None)
+ self.assertEqual(outcome.code if isinstance(outcome, ExperimentHealthFinding) else outcome, expected)
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 3fffbba109b6..82a85f5e4b94 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -97,7 +97,7 @@ def _raise(_ctx: HealthContext) -> Any:
MULTIPLE_VARIANTS_CHECK = next(check for check in EXPOSURE_HEALTH_CHECKS if check.key == Key.MULTIPLE_VARIANTS_SHARE)
-FAILING_CHECK = HealthCheck(key=Key.FLAG_STATE, name="Failing", find=_raise, skip=lambda _ctx: None)
+FAILING_CHECK = HealthCheck(key=Key.FLAG_STATE, name="Failing", run=_raise)
class TestEvaluate(TestCase):
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index cb62d3bf1e0b..edd7480bc924 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -17,6 +17,14 @@ from products.experiments.backend.health.checks.flag_change_findings import flag
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
from products.experiments.backend.health.context import FlagHistoryNotCheckedReason, HealthContext, parse_flag_state
+from products.experiments.backend.health.skips import (
+ FLAG_DELETED,
+ NEEDS_VARIANTS,
+ NOT_AFTER_END,
+ NOT_ON_ARCHIVED,
+ CheckSkip,
+ not_applicable,
+)
from products.experiments.backend.health.test.factories import ENDED, RUNNING, RUNNING_CONTEXT, Status, flag_history
from products.feature_flags.backend.facade.edits import SplitChange, parse_variants
@@ -83,7 +91,8 @@ def _context(
def _split_code_finding(ctx: HealthContext) -> ExperimentHealthFinding | None:
# The registry runs both checks, and a history the reader can't view gets the split check's code.
- return flag_history_not_readable(ctx) or split_changed_mid_run(ctx)
+ outcomes = (flag_history_not_readable(ctx), split_changed_mid_run(ctx))
+ return next((outcome for outcome in outcomes if isinstance(outcome, ExperimentHealthFinding)), None)
class TestSplitChangedMidRun(TestCase):
@@ -267,7 +276,7 @@ class TestSplitChangedMidRun(TestCase):
) -> None:
finding = split_changed_mid_run(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(finding.code, ExperimentHealthFindingCode.SPLIT_CHANGED_MID_RUN)
self.assertEqual((finding.subcode, finding.severity, finding.diagnostic_ref), expected)
self.assertEqual(finding.detail, expected_detail)
@@ -327,7 +336,7 @@ class TestSplitChangedMidRun(TestCase):
def test_evidence(self, _name: str, ctx: HealthContext, expected: dict[str, Any]) -> None:
finding = _split_code_finding(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(finding.evidence, expected)
@parameterized.expand(
@@ -396,7 +405,7 @@ class TestSplitChangedMidRun(TestCase):
) -> None:
finding = _split_code_finding(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(finding.code, finding.subcode, finding.severity, finding.diagnostic_ref, finding.title),
(
@@ -412,19 +421,24 @@ class TestSplitChangedMidRun(TestCase):
@parameterized.expand(
[
- ("ended", _context(EVEN, UNEVEN, status=ENDED)),
- ("archived", _context(EVEN, UNEVEN, archived=True)),
- ("deleted_flag", _context(EVEN, UNEVEN, deleted=True)),
- ("nobody_exposed_before_any_change", _context(EVEN)),
- ("flag_without_variants_reported_by_flag_variants_mismatch", _context(EVEN, [])),
+ ("ended", _context(EVEN, UNEVEN, status=ENDED), not_applicable(NOT_AFTER_END)),
+ ("archived", _context(EVEN, UNEVEN, archived=True), not_applicable(NOT_ON_ARCHIVED)),
+ ("deleted_flag", _context(EVEN, UNEVEN, deleted=True), not_applicable(FLAG_DELETED)),
+ ("nobody_exposed_before_any_change", _context(EVEN), None),
+ (
+ "flag_without_variants_reported_by_flag_variants_mismatch",
+ _context(EVEN, []),
+ not_applicable(NEEDS_VARIANTS),
+ ),
(
"not_checked_on_an_ended_experiment",
_context(EVEN, status=ENDED, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),
+ not_applicable(NOT_AFTER_END),
),
]
)
- def test_no_finding(self, _name: str, ctx: HealthContext) -> None:
- self.assertIsNone(_split_code_finding(ctx))
+ def test_no_finding(self, _name: str, ctx: HealthContext, expected: CheckSkip | None) -> None:
+ self.assertEqual((_split_code_finding(ctx), split_changed_mid_run(ctx)), (None, expected))
@parameterized.expand(
[
@@ -449,7 +463,7 @@ class TestSplitChangedMidRun(TestCase):
shipped = flag_state(ctx)
self.assertIsNone(split_changed_mid_run(ctx))
- assert shipped is not None
+ assert isinstance(shipped, ExperimentHealthFinding)
self.assertEqual(shipped.code, ExperimentHealthFindingCode.VARIANT_SHIPPED_WHILE_RUNNING)
self.assertEqual(
shipped.detail,
diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.py
index c0c4d0d235ae..49bc63529691 100644
--- a/products/experiments/backend/health/test/test_zero_exposures.py
+++ b/products/experiments/backend/health/test/test_zero_exposures.py
@@ -11,7 +11,7 @@ from parameterized import parameterized
from posthog.schema import MultipleVariantHandling
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingActionKind
+from products.experiments.backend.facade.contracts import ExperimentHealthFinding, ExperimentHealthFindingActionKind
from products.experiments.backend.health.checks.zero_exposures import (
CALLS_NOT_COUNTED,
CUSTOM_EXPOSURE_NOT_MATCHED,
@@ -19,15 +19,10 @@ from products.experiments.backend.health.checks.zero_exposures import (
FLAG_NOT_CALLED,
zero_exposures,
)
-from products.experiments.backend.health.context import (
- ExposureEvents,
- ExposureTotals,
- FlagState,
- FlagVariant,
- load_health_context,
-)
+from products.experiments.backend.health.context import ExposureEvents, ExposureTotals, FlagState, load_health_context
from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.facade.edits import FlagVariant
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -110,7 +105,7 @@ class TestZeroExposures(TestCase):
finding = zero_exposures(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual((finding.subcode, finding.actions), (subcode, actions))
@parameterized.expand(
@@ -139,7 +134,7 @@ class TestZeroExposures(TestCase):
finding = zero_exposures(ctx)
- assert finding is not None
+ assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual(
(
finding.title,