:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): give health findings advice that fits the cause (#129133)
Marcel Poelkerd1051a3875f1
diff --git a/products/experiments/backend/health/checks/bucketing_changed.py b/products/experiments/backend/health/checks/bucketing_changed.pyindex a3dbdf02d500..7759ce644dc5 100644--- a/products/experiments/backend/health/checks/bucketing_changed.py+++ b/products/experiments/backend/health/checks/bucketing_changed.py@@ -99,6 +99,20 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> CheckOutcome:title = _KEY_TITLE if effect.shape == BucketingChangeShape.KEY_CHANGED else _TITLEwhat_changed = [_what_changed(net, history.team_timezone) for net in effect.net_changes]+ if _continuity_turned_on(effect):+ # Turning persistence on keeps the variants that exposed users already have. Only a user who logs in again from+ # a new anonymous ID can get another one, and where an SDK can't apply persistence, nobody moves.+ return _finding(+ history,+ effect,+ severity=ExperimentHealthFindingSeverity.INFO,+ title=title,+ sentences=[+ *what_changed,+ "Turning it on doesn't move users who were already exposed to another variant, unless they log in "+ "later on another device or after logging out.",+ ],+ )if few is not None:return _finding(history,@@ -159,7 +173,7 @@ def _what_changed(change: BucketingChange, team_timezone: str) -> str:case BucketingSetting.AGGREGATION | BucketingSetting.BUCKETING_IDENTIFIER:return f"The flag changed from matching by {before} to matching by {after} on {at}."case BucketingSetting.EXPERIENCE_CONTINUITY:- return f'"Persist flag across authentication steps" was turned {after} on {at}.'+ return f'On {at}, "Persist flag across authentication steps" was turned {after}.'def _consequence(effect: BucketingNetEffect) -> list[str]:@@ -182,8 +196,13 @@ def _consequence(effect: BucketingNetEffect) -> list[str]:"Users whose device ID differs from their user ID got a new variant draw, so they may have switched ""variants.")+ case _ if effect.net_changes[0].after:+ consequence = "Users exposed between these changes may have switched variants."case _:- consequence = "Users who logged in got a new variant draw, so they may have switched variants."+ consequence = (+ "Users who logged in while it was on no longer keep the variant they got before logging in, so they "+ "may have switched variants."+ )return [consequence,"A sample ratio mismatch or users in more than one variant can come from "@@ -191,6 +210,16 @@ def _consequence(effect: BucketingNetEffect) -> list[str]:]+def _continuity_turned_on(effect: BucketingNetEffect) -> bool:+ """Whether one save turned persistence on and no other setting changed. A setting that changed and was set back,+ or persistence that was off for a while, can have moved exposed users in between."""+ return (+ effect.changed_settings == (BucketingSetting.EXPERIENCE_CONTINUITY,)+ and effect.change_count == 1+ and bool(effect.net_changes[0].after)+ )++def _key_state(effect: BucketingNetEffect) -> list[str]:if effect.shape != BucketingChangeShape.KEY_CHANGED:return []diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.pyindex 75b03bb25960..9c8c12c1f227 100644--- a/products/experiments/backend/health/checks/forced_variant.py+++ b/products/experiments/backend/health/checks/forced_variant.py@@ -40,8 +40,9 @@ from products.experiments.backend.health.split_changes import format_change_timefrom products.feature_flags.backend.facade.edits import ConditionsChange# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.-# Every condition that includes users pins a variant, so no user is randomly assigned. One variant gets-# everyone, or each variant gets a different group of users, so the results compare nothing valid.+# Every condition that includes users pins a variant, and one of them includes everyone it reaches, so no user is+# randomly assigned. One variant gets everyone, or each variant gets a different group of users, so the results+# compare nothing valid.ALL_CONDITIONS_PINNED = "all_conditions_pinned"# Some conditions pin a variant. The users they match count in that variant's results without random# assignment, so they bias the comparison unless the analysis excludes them, for example as test users.@@ -49,6 +50,9 @@ SOME_CONDITIONS_PINNED = "some_conditions_pinned"# Every condition that includes users pins the variant that one property names, so the code that sets the# property assigns the variants. The comparison is valid if that code assigns at random, which PostHog can't see.ASSIGNED_BY_PROPERTY = "assigned_by_property"+# Every condition that includes users pins a variant and matches only some users, so only those users enter the+# experiment and everyone else gets no variant. Usually a test set-up, such as a list of the team's own users.+CLOSED_TO_MATCHED_USERS = "closed_to_matched_users"@frozen@@ -108,6 +112,17 @@ def forced_variant_release_condition(ctx: HealthContext) -> CheckOutcome:diagnostic_ref="A7b",grade_evidence={"variant_property": variant_property.key, **set_after_launch.evidence},)+ if not any(condition.includes_everyone for condition in pins.pinned):+ return _finding(+ subcode=CLOSED_TO_MATCHED_USERS,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title=f"Only users who match {set_names(pinned_numbers)} get a variant",+ detail=_closed_detail(pinned_numbers, pinned_keys, is_draft, set_after_launch),+ pinned_numbers=pinned_numbers,+ pinned_keys=pinned_keys,+ diagnostic_ref="A7b",+ grade_evidence=set_after_launch.evidence,+ )return _finding(subcode=ALL_CONDITIONS_PINNED,severity=ExperimentHealthFindingSeverity.WARNING,@@ -247,12 +262,13 @@ def _all_pinned_detail(f"Every release condition that includes users sets a variant override ({_quoted(pinned_keys)}), so "f"{when}each user gets the variant of the condition they match instead of a random one.")+ # The conditions can assign by segment on purpose, so the fix is offered for a random assignment.why = ("Each variant then holds a different group of users, so a difference in the results can come from ""who those users are and not from the variant.")fix = (- f"Remove {overrides} so users are assigned at random. Users who already saw a variant can then "+ f"To assign users at random, remove {overrides}. Users who already saw a variant can then "f"switch to another one. {set_after_launch.remedy_after_removal()}")if is_draft:@@ -260,6 +276,50 @@ def _all_pinned_detail(return f"{fact}{set_after_launch.sentence} {why} {fix}"+def _closed_detail(+ numbers: Sequence[int], keys: Sequence[str], is_draft: bool, set_after_launch: _SetAfterLaunch+) -> str:+ enter = "after launch only those users enter" if is_draft else "only those users enter"+ if len(keys) == 1:+ gets = f'sets the variant override "{keys[0]}", so {enter} the experiment, and they all get "{keys[0]}".'+ why = (+ "Everyone else gets no variant. With no other variant to compare against, the results can't show "+ f'whether "{keys[0]}" makes a difference.'+ )+ else:+ gets = (+ f"sets a variant override ({_quoted(keys)}), so {enter} the experiment, and each gets the variant of "+ "their condition."+ )+ why = (+ "Everyone else gets no variant. Each variant holds a different group of users, so a difference in the "+ "results can come from who those users are and not from the variant."+ )+ if len(numbers) == 1:+ fact = (+ f"Set {numbers[0]} is the only release condition that includes users. It matches only some users and {gets}"+ )+ else:+ fact = (+ f"{set_names(numbers)} are the only release conditions that include users. Each matches only some users "+ f"and {gets}"+ )+ overrides = "the override" if len(numbers) == 1 else "the overrides"+ if is_draft:+ fix = (+ "An override helps to test the experiment before launch. Before you launch, add a release condition "+ f"without a variant override for the users you want to test. If these are those users, remove {overrides} "+ "instead."+ )+ else:+ fix = (+ "If these are test users, add a release condition without a variant override for the users you want to "+ f"test, or end the experiment. If they are the users you want to test, remove {overrides} so they are "+ f"assigned at random. {set_after_launch.remedy_after_removal()}"+ )+ return f"{fact}{set_after_launch.sentence} {why} {fix}"++def _assigned_by_property_detail(variant_property: str, set_after_launch: str) -> str:return (f'Each release condition that includes users matches the users whose property "{variant_property}" holds a 'diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.pyindex d89c8c78c597..c8c1dd67e232 100644--- a/products/experiments/backend/health/checks/zero_exposures.py+++ b/products/experiments/backend/health/checks/zero_exposures.py@@ -59,7 +59,10 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:# deletion.if flag is None or flag.deleted:return _cause_unknown(evidence, unit, ended=ctx.has_ended)- if ctx.is_paused:+ events = ctx.exposure_events+ # A flag that no code called since launch exposes nobody after a resume either, so the missing calls are the cause.+ never_called = events is not None and events.required_flag_key is not None and ctx.flag_called_during_run is False+ if ctx.is_paused and not never_called:# Info, because the flag-state check already reports the pause, and a pause needs no fix.return _finding(subcode=EXPERIMENT_PAUSED,@@ -72,7 +75,6 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:evidence=evidence,actions=(),)- events = ctx.exposure_eventsif events is None:return _cause_unknown(evidence, unit, ended=ctx.has_ended)if events.required_flag_key is None:@@ -132,6 +134,10 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:"example with getFeatureFlag in the browser or with get_flag on the result of evaluate_flags in Python. "f"{no_call_recorded} New calls can take up to {LAST_CALLED_AT_LAG} to show up here.")+ if ctx.is_paused:+ detail += (+ " The experiment is also paused, and resuming it exposes nobody until your code reads the flag."+ )# Info after the end, because no change to the code can expose more users then, so the finding only explains# the empty results.return _finding(@@ -198,19 +204,31 @@ def _test_accounts_filtered_out() -> ExperimentHealthFinding:exposed_units = f"the only {unit}" if exposed == 1 else f"all {exposed:,} {unit}s"served = f'{exposed_units} that got a variant of the feature flag "{flag_key}"'- fix = 'Turn off "Filter out internal and test users" in the exposure criteria'+ turn_off = '"Filter out internal and test users" in the exposure criteria'+ # When the filters match every user who got a variant, usually only the team's own users got one, and the filter+ # then works as intended.if ended:title = f"No {unit}s exposed: the test account filter removed all of them"detail = (f"No {unit}s were exposed while the experiment ran, because the exposure criteria filter out test "- f"accounts, and the project's test account filters matched {served}. {fix} to count them."+ f"accounts, and the project's test account filters matched {served}. If these were your own team's "+ f"{unit}s, the experiment didn't reach other {unit}s. Otherwise turn off {turn_off} to count them."+ )+ actions: tuple[ExperimentHealthFindingActionKind, ...] = (+ ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,)else:title = f"No {unit}s exposed: the test account filter removes all of them"detail = (f"The exposure criteria filter out test accounts, and the project's test account filters match {served} "- f"during the experiment, so nobody is counted as exposed. {fix}, or change the project's test account "- f"filters so that they don't match your {unit}s."+ "during the experiment, so nobody is counted as exposed. If these are your own team's "+ f"{unit}s, the experiment hasn't reached other {unit}s yet. The code that reads the flag may only run in "+ "development, or the release conditions may include only your team. Otherwise change the project's test "+ f"account filters so that they don't match your {unit}s, or turn off {turn_off}."+ )+ actions = (+ ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG,+ ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,)return _finding(subcode=CALLS_NOT_COUNTED,@@ -219,7 +237,7 @@ def _test_accounts_filtered_out(title=title,detail=detail,evidence=evidence,- actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),+ actions=actions,)diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex 37b1cd7f61f2..eb3446afc093 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -180,6 +180,9 @@ class ReachableCondition:variant_property: VariantProperty | None# The cohorts whose members the condition matches. A `not_in` cohort filter matches the users outside the cohort.member_cohort_ids: frozenset[int] = frozenset()+ # Whether the condition has no filters, or only filters that leave users out, so it includes nearly everyone who+ # reaches it.+ includes_everyone: bool = False@frozen@@ -673,6 +676,30 @@ def _member_cohort_ids(condition: Mapping[str, Any]) -> frozenset[int]:)+_EXCLUDING_OPERATORS = frozenset(+ {+ PropertyOperator.IS_NOT,+ PropertyOperator.NOT_ICONTAINS,+ PropertyOperator.NOT_ICONTAINS_MULTI,+ PropertyOperator.NOT_STARTS_WITH,+ PropertyOperator.NOT_ENDS_WITH,+ PropertyOperator.NOT_REGEX,+ PropertyOperator.NOT_BETWEEN,+ PropertyOperator.NOT_IN,+ PropertyOperator.IS_NOT_SET,+ PropertyOperator.SEMVER_NEQ,+ PropertyOperator.IS_CLEANED_PATH_NOT_EXACT,+ }+)+++def _includes_everyone(condition: Mapping[str, Any]) -> bool:+ properties = condition.get("properties")+ if not isinstance(properties, list) or not properties:+ return True+ return all(isinstance(prop, dict) and prop.get("operator") in _EXCLUDING_OPERATORS for prop in properties)++def _reachable_condition(number: int, condition: Mapping[str, Any], variant_keys: set[str], flag_aggregation: int | None) -> ReachableCondition:@@ -682,6 +709,7 @@ def _reachable_condition(pinned_variant=pinned,variant_property=None if pinned is None else _variant_property(condition, pinned, flag_aggregation),member_cohort_ids=_member_cohort_ids(condition),+ includes_everyone=_includes_everyone(condition),)diff --git a/products/experiments/backend/health/exposure_inputs.py b/products/experiments/backend/health/exposure_inputs.pyindex 16aebe024c6e..d6a15e82d67b 100644--- a/products/experiments/backend/health/exposure_inputs.py+++ b/products/experiments/backend/health/exposure_inputs.py@@ -13,6 +13,7 @@ from posthog.models.user import Userfrom products.experiments.backend.facade.contracts import HealthExposureInputsStatusfrom products.experiments.backend.health.context import ExposureTotals+from products.experiments.backend.health.launch_time import hours_since_launchfrom products.experiments.backend.hogql_queries.experiment_exposures_query_runner import (ExperimentExposuresQueryRunner,build_experiment_exposure_query,@@ -40,7 +41,9 @@ class ExposureInputs:error: str | None-def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_query_events: bool) -> ExposureInputs:+def load_exposure_inputs(+ experiment: Experiment, *, user: User | None, can_query_events: bool, can_read_flag_history: bool+) -> ExposureInputs:"""Reads the cached answer of the experiment's exposure query. A missing or stale answer starts a calculation in thebackground instead of blocking, because the query can run for minutes. Never raises.@@ -95,7 +98,11 @@ def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_querysample_ratio_expected={},daily_exposures={},# The checks read the hours at computation, not now.- hours_since_launch=(computed_at - experiment.start_date).total_seconds() / 3600 if computed_at else None,+ hours_since_launch=(+ hours_since_launch(experiment, computed_at, can_read_flag_history=can_read_flag_history)+ if computed_at+ else None+ ),counts_groups=aggregates_by_group(runner.group_type_index),filters_test_accounts=applies_test_account_filters(experiment.team, query.exposure_criteria),has_positive_test_account_filter=has_positive_test_account_filter(experiment.team),diff --git a/products/experiments/backend/health/launch_time.py b/products/experiments/backend/health/launch_time.pynew file mode 100644index 000000000000..d80ec7757de5--- /dev/null+++ b/products/experiments/backend/health/launch_time.py@@ -0,0 +1,49 @@+from datetime import datetime++from posthog.exceptions_capture import capture_exception+from posthog.models.activity_logging.activity_log import ActivityLog++from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS+from products.experiments.backend.models.experiment import Experiment+++def hours_since_launch(experiment: Experiment, at: datetime, *, can_read_flag_history: bool) -> float | None:+ """+ The hours from the experiment's launch to `at`, which the exposure checks wait on. The start date can be moved+ back to before the launch, when the flag didn't serve variants yet, so the wait counts from the later of the start+ date and the latest launch in the activity log. The activity log needs the access of the flag history, so a reader+ without it waits from the start date.+ """+ start_date = experiment.start_date+ if start_date is None:+ return None+ hours = (at - start_date).total_seconds() / 3600+ # A wait that holds by the start date also holds by the launch, which can only be later.+ if hours < ZERO_EXPOSURES_GRACE_HOURS or not can_read_flag_history:+ return hours+ launched_at = _latest_launch(experiment)+ if launched_at is None or launched_at <= start_date:+ return hours+ return (at - launched_at).total_seconds() / 3600+++def _latest_launch(experiment: Experiment) -> datetime | None:+ # A launch logs the start date that it sets on a draft. An experiment created with a start date logs none, and its+ # start date stands.+ try:+ return (+ ActivityLog.objects.filter(+ team_id=experiment.team_id,+ scope="Experiment",+ item_id=str(experiment.id),+ activity="updated",+ detail__contains={"changes": [{"field": "start_date", "before": None}]},+ )+ .order_by("-created_at")+ .values_list("created_at", flat=True)+ .first()+ )+ except Exception as error:+ # The start date is a safe fallback, so a failed lookup must not fail the exposure answer.+ capture_exception(error)+ return Nonediff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex 7c5d4d8c0d7e..a285f9598cc4 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -47,7 +47,12 @@ def load_experiment_health_inputs(experiment = _load_experiment(team_id, experiment_id)return HealthInputs(context=_health_context(experiment, reader_access, user),- exposures=load_exposure_inputs(experiment, user=user, can_query_events=reader_access.can_query_events),+ exposures=load_exposure_inputs(+ experiment,+ user=user,+ can_query_events=reader_access.can_query_events,+ can_read_flag_history=reader_access.can_read_flag_history,+ ),reader_access=reader_access,flag_version=experiment.feature_flag.version if experiment.feature_flag_id else None,project=build_project_setup_facts(diff --git a/products/experiments/backend/health/test/test_bucketing_changed.py b/products/experiments/backend/health/test/test_bucketing_changed.pyindex 49b9222f64a7..19cdf6144253 100644--- a/products/experiments/backend/health/test/test_bucketing_changed.py+++ b/products/experiments/backend/health/test/test_bucketing_changed.py@@ -29,6 +29,7 @@ LAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)FIRST_CHANGE_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)FIRST_CHANGE = "September 12, 2026 at 3:04 PM CEST"SECOND_CHANGE = "September 12, 2026 at 3:21 PM CEST"+THIRD_CHANGE = "September 12, 2026 at 3:38 PM CEST"MIXED_ASSIGNMENTS = "A sample ratio mismatch or users in more than one variant can come from this change."EVEN = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]KEY = BucketingSetting.KEY@@ -134,9 +135,38 @@ class TestBucketingChangedMidRun(TestCase):ExperimentHealthFindingSeverity.WARNING,"The flag's variant assignment changed after launch",),- f'"Persist flag across authentication steps" was turned off on {FIRST_CHANGE}. The change came 3 '- "hours after launch. Users who logged in got a new variant draw, so they may have switched variants. "- f"{MIXED_ASSIGNMENTS} Reset the experiment, or move its start date to {FIRST_CHANGE}.",+ f'On {FIRST_CHANGE}, "Persist flag across authentication steps" was turned off. The change came 3 '+ "hours after launch. Users who logged in while it was on no longer keep the variant they got before "+ f"logging in, so they may have switched variants. {MIXED_ASSIGNMENTS} Reset the experiment, or move "+ f"its start date to {FIRST_CHANGE}.",+ ),+ (+ "experience_continuity_turned_on_after_a_set_back_change",+ _context(+ (IDENTIFIER, "distinct_id", "device_id"),+ (IDENTIFIER, "device_id", "distinct_id"),+ (CONTINUITY, False, True),+ ),+ (+ "experience_continuity_changed",+ ExperimentHealthFindingSeverity.WARNING,+ "The flag's variant assignment changed after launch",+ ),+ f'On {THIRD_CHANGE}, "Persist flag across authentication steps" was turned on. Users exposed between '+ "these changes may have switched variants. A sample ratio mismatch or users in more than one variant "+ f"can come from these changes. Reset the experiment, or move its start date to {THIRD_CHANGE}.",+ ),+ (+ "experience_continuity_turned_on",+ _context((CONTINUITY, False, True), exposed_units=5),+ (+ "experience_continuity_changed",+ ExperimentHealthFindingSeverity.INFO,+ "The flag's variant assignment changed after launch",+ ),+ f'On {FIRST_CHANGE}, "Persist flag across authentication steps" was turned on. Turning it on '+ "doesn't move users who were already exposed to another variant, unless they log in later on another "+ "device or after logging out.",),("aggregation_changed",@@ -154,8 +184,8 @@ class TestBucketingChangedMidRun(TestCase):"two_settings_name_the_key_first",_context((CONTINUITY, True, False), (KEY, "old-key", "new-key")),("key_changed", ExperimentHealthFindingSeverity.WARNING, "The flag key changed after launch"),- f'The flag key changed from "old-key" to "new-key" on {SECOND_CHANGE}. "Persist flag across '- f'authentication steps" was turned off on {FIRST_CHANGE}. The key is an input of the variant draw, so '+ f'The flag key changed from "old-key" to "new-key" on {SECOND_CHANGE}. On {FIRST_CHANGE}, "Persist '+ 'flag across authentication steps" was turned off. The key is an input of the variant draw, so ''every user got a new one. Exposures under "old-key" are no longer in the results, and code that ''still uses "old-key" no longer gets this flag. Reset the experiment, or move its start date to 'f"{SECOND_CHANGE}.",diff --git a/products/experiments/backend/health/test/test_exposure_inputs.py b/products/experiments/backend/health/test/test_exposure_inputs.pyindex 388252197d89..1c1a679f6a75 100644--- a/products/experiments/backend/health/test/test_exposure_inputs.py+++ b/products/experiments/backend/health/test/test_exposure_inputs.py@@ -108,7 +108,7 @@ class TestLoadExposureInputs(BaseTest):"products.experiments.backend.health.exposure_inputs.ExperimentExposuresQueryRunner.run",return_value=response,):- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)self.assertEqual((inputs.status, inputs.error), (expected_status, CALCULATION_ERROR if refresh_failed else None)@@ -144,7 +144,7 @@ class TestLoadExposureInputs(BaseTest):exposure_criteria={"multiple_variant_handling": "not_a_handling"},)- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)self.assertEqual(inputs.status, HealthExposureInputsStatus.FAILED)@@ -152,7 +152,7 @@ class TestLoadExposureInputs(BaseTest):experiment = self._create_experiment("exposure-inputs-failed-calculation")cache_key = self._store_failed_calculation(experiment)- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)self.assertEqual((inputs.status, inputs.error), (HealthExposureInputsStatus.FAILED, CALCULATION_ERROR))self.assertEqual(get_query_status(team_id=self.team.pk, query_id=cache_key).error_message, CALCULATION_ERROR)diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.pyindex 3185c8502024..deda497d2d00 100644--- a/products/experiments/backend/health/test/test_forced_variant.py+++ b/products/experiments/backend/health/test/test_forced_variant.py@@ -43,6 +43,14 @@ IOS: dict[str, Any] = {"properties": [{"key": "$device_type", "value": "iOS", "operator": "exact", "type": "person"}],"rollout_percentage": 100,}+NOT_STAFF: dict[str, Any] = {+ "properties": [{"key": "email", "value": "@example.com", "operator": "not_icontains", "type": "person"}],+ "rollout_percentage": 100,+}+NOT_IOS: dict[str, Any] = {+ "properties": [{"key": "$device_type", "value": "iOS", "operator": "is_not", "type": "person"}],+ "rollout_percentage": 100,+}CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)@@ -128,6 +136,16 @@ class TestForcedVariantReleaseCondition(TestCase):("cohorts_forced_into_both_variants",_context([{**IOS, "variant": "test"}, {**STAFF, "variant": "control"}]),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test, control", None),+ ),+ (+ "everyone_but_staff_forced_into_one_variant",+ _context([{**NOT_STAFF, "variant": "test"}]),+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "test", None),+ ),+ (+ "segment_and_its_complement_forced_into_both_variants",+ _context([{**IOS, "variant": "test"}, {**NOT_IOS, "variant": "control"}]),("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test, control", None),),(@@ -148,7 +166,7 @@ class TestForcedVariantReleaseCondition(TestCase):{**_property_is("plan", "pro"), "variant": "test"},]),- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),),("variants_mapped_from_a_person_and_a_group_property_of_one_key",@@ -158,12 +176,27 @@ class TestForcedVariantReleaseCondition(TestCase):{**_property_is("arm", "test", group_type_index=0), "variant": "test"},]),- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),),("one_variant_mapped_from_a_property",_context([{**_property_is("arm", "control"), "variant": "control"}]),- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),+ ),+ (+ "list_is_the_only_condition_with_users",+ _context([{**STAFF, "variant": "test"}, {**EVERYONE, "rollout_percentage": 0}]),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1", "test", None),+ ),+ (+ "draft_list_is_the_only_condition_with_users",+ _context([{**STAFF, "variant": "test"}, {**IOS, "variant": "test"}], status=DRAFT),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test", None),+ ),+ (+ "list_and_catch_all_pin_one_variant",+ _context([{**STAFF, "variant": "test"}, {**EVERYONE, "variant": "test"}]),+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test", None),),("variants_mapped_from_two_properties",@@ -173,7 +206,7 @@ class TestForcedVariantReleaseCondition(TestCase):{**_property_is("cell", "test"), "variant": "test"},]),- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),),("override_after_a_catch_all_is_unreachable", _context([EVERYONE, {**STAFF, "variant": "test"}]), None),(@@ -406,6 +439,54 @@ class TestForcedVariantReleaseCondition(TestCase):))+ @parameterized.expand(+ [+ (+ "one_list",+ [{**STAFF, "variant": "test"}, {**EVERYONE, "rollout_percentage": 0}],+ "Only users who match Set 1 get a variant",+ "Set 1 is the only release condition that includes users. It matches only some users and sets the "+ 'variant override "test", so only those users enter the experiment, and they all get "test". Everyone '+ "else gets no variant. With no other variant to compare against, the results can't show whether "+ '"test" makes a difference. If these are test users, add a release condition without a variant '+ "override for the users you want to test, or end the experiment. If they are the users you want to "+ "test, remove the override so they are assigned at random. Then reset the experiment, or move its "+ "start date to after the removal.",+ ),+ (+ "lists_for_both_variants",+ [{**STAFF, "variant": "test"}, {**IOS, "variant": "control"}],+ "Only users who match Sets 1 and 2 get a variant",+ "Sets 1 and 2 are the only release conditions that include users. Each matches only some users and "+ 'sets a variant override ("test", "control"), so only those users enter the experiment, and each gets '+ "the variant of their condition. Everyone else gets no variant. Each variant holds a different group "+ "of users, so a difference in the results can come from who those users are and not from the "+ "variant. If these are test users, add a release condition without a variant override for the users "+ "you want to test, or end the experiment. If they are the users you want to test, remove the "+ "overrides so they are assigned at random. Then reset the experiment, or move its start date to after "+ "the removal.",+ ),+ (+ "segment_and_its_complement",+ [{**IOS, "variant": "test"}, {**NOT_IOS, "variant": "control"}],+ "No users are randomly assigned to a variant",+ 'Every release condition that includes users sets a variant override ("test", "control"), so each '+ "user gets the variant of the condition they match instead of a random one. Each variant then holds a "+ "different group of users, so a difference in the results can come from who those users are and not "+ "from the variant. To assign users at random, remove the overrides. Users who already saw a variant "+ "can then switch to another one. Then reset the experiment, or move its start date to after the "+ "removal.",+ ),+ ]+ )+ def test_names_who_can_enter_when_every_condition_pins(+ self, _name: str, groups: list[dict[str, Any]], title: str, detail: str+ ) -> None:+ finding = forced_variant_release_condition(_context(groups))++ assert isinstance(finding, ExperimentHealthFinding)+ self.assertEqual((finding.title, finding.detail), (title, detail))+def test_variants_mapped_from_a_property_say_who_assigns_them(self) -> None:finding = forced_variant_release_condition(_context(diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.pyindex 33a766c8f645..163a1b4c6f79 100644--- a/products/experiments/backend/health/test/test_zero_exposures.py+++ b/products/experiments/backend/health/test/test_zero_exposures.py@@ -92,11 +92,25 @@ class TestZeroExposures(TestCase):"flag_called_by_test_accounts_only",{"flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},CALLS_NOT_COUNTED,- (Action.EDIT_EXPOSURE_CRITERIA,),+ (Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),WARNING,),("calls_unknown", {"flag_called_during_run": None}, None, (Action.EDIT_EXPOSURE_CRITERIA,), WARNING),("paused", {"is_paused": True, "flag_called_during_run": True}, EXPERIMENT_PAUSED, (), INFO),+ (+ "paused_flag_not_called",+ {"is_paused": True, "flag_called_during_run": False},+ FLAG_NOT_CALLED,+ (),+ WARNING,+ ),+ (+ "paused_custom_event",+ {"is_paused": True, "exposure_events": CUSTOM_EVENT, "flag_called_during_run": False},+ EXPERIMENT_PAUSED,+ (),+ INFO,+ ),("custom_event_whatever_the_flag_calls",{"exposure_events": CUSTOM_EVENT, "flag_called_during_run": False},@@ -173,6 +187,10 @@ class TestZeroExposures(TestCase):assert isinstance(finding, ExperimentHealthFinding)self.assertEqual((finding.subcode, finding.actions, finding.severity), (subcode, actions, severity))self.assertEqual(finding.detail.startswith("No users were exposed while the experiment ran"), ctx.has_ended)+ self.assertEqual(+ finding.detail.endswith("resuming it exposes nobody until your code reads the flag."),+ ctx.is_paused and subcode == FLAG_NOT_CALLED,+ )@parameterized.expand([@@ -221,8 +239,11 @@ class TestZeroExposures(TestCase):"No users exposed: the test account filter removes all of them","The exposure criteria filter out test accounts, and the project's test account filters match all 1,200 "'users that got a variant of the feature flag "checkout-test" during the experiment, so nobody is '- 'counted as exposed. Turn off "Filter out internal and test users" in the exposure criteria, or change '- "the project's test account filters so that they don't match your users.",+ "counted as exposed. If these are your own team's users, the experiment hasn't reached other users "+ "yet. The code that reads the flag may only run in development, or the release conditions may include "+ "only your team. Otherwise change the project's test account filters so "+ 'that they don\'t match your users, or turn off "Filter out internal and test users" in the exposure '+ "criteria.",),("ended_one_user",@@ -233,8 +254,9 @@ class TestZeroExposures(TestCase):"No users exposed: the test account filter removed all of them","No users were exposed while the experiment ran, because the exposure criteria filter out test ""accounts, and the project's test account filters matched the only user that got a variant of the "- 'feature flag "checkout-test". Turn off "Filter out internal and test users" in the exposure criteria '- "to count them.",+ "feature flag \"checkout-test\". If these were your own team's users, the experiment didn't reach "+ 'other users. Otherwise turn off "Filter out internal and test users" in the exposure criteria to '+ "count them.",),("nobody_without_the_filter",diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex ba8031444be7..6814d8dcf7b8 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -73,6 +73,7 @@ from products.experiments.backend.health.context import (load_health_context,)from products.experiments.backend.health.exposure_gate import load_split_history+from products.experiments.backend.health.launch_time import hours_since_launchfrom products.experiments.backend.health.registry import (EXPOSURE_HEALTH_CHECKS,evaluate as evaluate_health,@@ -582,7 +583,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):sample_ratio_mismatch_p_value=sample_ratio_mismatch.p_value if sample_ratio_mismatch else None,sample_ratio_expected=sample_ratio_mismatch.expected if sample_ratio_mismatch else {},daily_exposures=daily_exposures,- hours_since_launch=self.experiment.hours_since_launch,+ hours_since_launch=hours_since_launch(+ self.experiment, datetime.now(UTC), can_read_flag_history=self._can_read_flag_history()+ ),counts_groups=aggregates_by_group(self.group_type_index),filters_test_accounts=applies_test_account_filters(self.team, self.exposure_criteria),has_positive_test_account_filter=has_positive_test_account_filter(self.team),diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex 6ff0cc4fe9bf..721c59d115cf 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -1,4 +1,5 @@import re+from contextlib import nullcontextfrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom zoneinfo import ZoneInfo@@ -7,6 +8,7 @@ import time_machinefrom posthog.test.base import _create_event, _create_person, flush_persons_and_events, snapshot_clickhouse_queriesfrom unittest.mock import patch+from django.db import OperationalErrorfrom django.forms.models import model_to_dictfrom django.test import override_settings@@ -2000,19 +2002,46 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):@parameterized.expand([- ("first_day", "2024-01-01T23:00:00Z", None, []),+ ("first_day", "2024-01-01T23:00:00Z", None, None, []),("after_a_day","2024-01-02T01:00:00Z",None,+ None,[(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],),("after_a_day_with_flag_calls","2024-01-02T01:00:00Z",datetime(2024, 1, 1, 12, tzinfo=UTC),+ None,[(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "calls_not_counted")],),+ ("first_day_after_a_launch_with_an_earlier_start_date", "2024-01-02T01:00:00Z", None, 12, []),+ (+ "after_a_day_from_a_launch_with_an_earlier_start_date",+ "2024-01-02T13:00:00Z",+ None,+ 12,+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],+ ),+ (+ "launch_hidden_from_a_reader_without_the_flag_history",+ "2024-01-02T01:00:00Z",+ None,+ 12,+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],+ False,+ ),+ (+ "launch_lookup_fails",+ "2024-01-02T01:00:00Z",+ None,+ 12,+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],+ True,+ True,+ ),])def test_zero_exposures_finding_waits_a_day_and_names_the_flag_calls(@@ -2020,10 +2049,23 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):_name: str,now: str,last_called_at: datetime | None,+ launched_hours_after_start: int | None,expected: list[tuple[ExperimentExposureHealthFindingCode, str]],+ reads_flag_history: bool = True,+ launch_lookup_fails: bool = False,) -> None:self.feature_flag.last_called_at = last_called_atself.feature_flag.save()+ if launched_hours_after_start is not None:+ launched_at = datetime(2024, 1, 1, tzinfo=UTC) + timedelta(hours=launched_hours_after_start)+ ActivityLog.objects.create(+ team_id=self.team.id,+ scope="Experiment",+ item_id=str(self.experiment.id),+ activity="updated",+ detail={"changes": [{"field": "start_date", "before": None, "after": launched_at.isoformat()}]},+ created_at=launched_at,+ )query = ExperimentExposureQuery(kind="ExperimentExposureQuery",experiment_id=self.experiment.id,@@ -2034,8 +2076,14 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):exposure_criteria=self.experiment.exposure_criteria,)- with time_machine.travel(now, tick=False):- response = ExperimentExposuresQueryRunner(team=self.team, query=query).calculate()+ lookup = patch(+ "products.experiments.backend.health.launch_time.ActivityLog.objects.filter",+ side_effect=OperationalError("the activity log is unavailable"),+ )+ with time_machine.travel(now, tick=False), lookup if launch_lookup_fails else nullcontext():+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=query, user=self.user if reads_flag_history else None+ ).calculate()self.assertEqual(response.total_exposures, {"control": 0, "test": 0})self.assertEqual([(finding.code, finding.subcode) for finding in response.health_findings or []], expected)diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex ad935fe6ac68..f7eb63cd6900 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -1057,7 +1057,10 @@ class EnterpriseExperimentsViewSet(`running_but_no_rollout` when 10 or more people got a variant from flag calls without aflag version that are no local evaluation, from an hour after the window starts. Whenevery condition that includes users pins the variant whose key one property holds,- `assigned_by_property` replaces `all_conditions_pinned`.+ `assigned_by_property` replaces `all_conditions_pinned`. When none of them includes+ everyone, so that every other user gets no variant, `closed_to_matched_users` replaces it,+ as a warning. A condition without filters, or with only filters that leave users out,+ includes everyone.A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The recordis the flag's `last_called_at`, which a periodic task updates, so it trails a call. Thediff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex c51e981ffff3..4f494329bc92 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -813,7 +813,10 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* `running_but_no_rollout` when 10 or more people got a variant from flag calls without a* flag version that are no local evaluation, from an hour after the window starts. When* every condition that includes users pins the variant whose key one property holds,- * `assigned_by_property` replaces `all_conditions_pinned`.+ * `assigned_by_property` replaces `all_conditions_pinned`. When none of them includes+ * everyone, so that every other user gets no variant, `closed_to_matched_users` replaces it,+ * as a warning. A condition without filters, or with only filters that leave users out,+ * includes everyone.** A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The record* is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The