Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): give health findings advice that fits the cause (#129133)

Marcel Poelkerd1051a3875f1
diff --git a/products/experiments/backend/health/checks/bucketing_changed.py b/products/experiments/backend/health/checks/bucketing_changed.py
index a3dbdf02d500..7759ce644dc5 100644
--- a/products/experiments/backend/health/checks/bucketing_changed.py
+++ b/products/experiments/backend/health/checks/bucketing_changed.py
@@ -99,6 +99,20 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> CheckOutcome:
title = _KEY_TITLE if effect.shape == BucketingChangeShape.KEY_CHANGED else _TITLE
what_changed = [_what_changed(net, history.team_timezone) for net in effect.net_changes]
+ if _continuity_turned_on(effect):
+ # Turning persistence on keeps the variants that exposed users already have. Only a user who logs in again from
+ # a new anonymous ID can get another one, and where an SDK can't apply persistence, nobody moves.
+ return _finding(
+ history,
+ effect,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title=title,
+ sentences=[
+ *what_changed,
+ "Turning it on doesn't move users who were already exposed to another variant, unless they log in "
+ "later on another device or after logging out.",
+ ],
+ )
if few is not None:
return _finding(
history,
@@ -159,7 +173,7 @@ def _what_changed(change: BucketingChange, team_timezone: str) -> str:
case BucketingSetting.AGGREGATION | BucketingSetting.BUCKETING_IDENTIFIER:
return f"The flag changed from matching by {before} to matching by {after} on {at}."
case BucketingSetting.EXPERIENCE_CONTINUITY:
- return f'"Persist flag across authentication steps" was turned {after} on {at}.'
+ return f'On {at}, "Persist flag across authentication steps" was turned {after}.'
def _consequence(effect: BucketingNetEffect) -> list[str]:
@@ -182,8 +196,13 @@ def _consequence(effect: BucketingNetEffect) -> list[str]:
"Users whose device ID differs from their user ID got a new variant draw, so they may have switched "
"variants."
)
+ case _ if effect.net_changes[0].after:
+ consequence = "Users exposed between these changes may have switched variants."
case _:
- consequence = "Users who logged in got a new variant draw, so they may have switched variants."
+ consequence = (
+ "Users who logged in while it was on no longer keep the variant they got before logging in, so they "
+ "may have switched variants."
+ )
return [
consequence,
"A sample ratio mismatch or users in more than one variant can come from "
@@ -191,6 +210,16 @@ def _consequence(effect: BucketingNetEffect) -> list[str]:
]
+def _continuity_turned_on(effect: BucketingNetEffect) -> bool:
+ """Whether one save turned persistence on and no other setting changed. A setting that changed and was set back,
+ or persistence that was off for a while, can have moved exposed users in between."""
+ return (
+ effect.changed_settings == (BucketingSetting.EXPERIENCE_CONTINUITY,)
+ and effect.change_count == 1
+ and bool(effect.net_changes[0].after)
+ )
+
+
def _key_state(effect: BucketingNetEffect) -> list[str]:
if effect.shape != BucketingChangeShape.KEY_CHANGED:
return []
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index 75b03bb25960..9c8c12c1f227 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -40,8 +40,9 @@ from products.experiments.backend.health.split_changes import format_change_time
from products.feature_flags.backend.facade.edits import ConditionsChange
# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.
-# Every condition that includes users pins a variant, so no user is randomly assigned. One variant gets
-# everyone, or each variant gets a different group of users, so the results compare nothing valid.
+# Every condition that includes users pins a variant, and one of them includes everyone it reaches, so no user is
+# randomly assigned. One variant gets everyone, or each variant gets a different group of users, so the results
+# compare nothing valid.
ALL_CONDITIONS_PINNED = "all_conditions_pinned"
# Some conditions pin a variant. The users they match count in that variant's results without random
# assignment, so they bias the comparison unless the analysis excludes them, for example as test users.
@@ -49,6 +50,9 @@ SOME_CONDITIONS_PINNED = "some_conditions_pinned"
# Every condition that includes users pins the variant that one property names, so the code that sets the
# property assigns the variants. The comparison is valid if that code assigns at random, which PostHog can't see.
ASSIGNED_BY_PROPERTY = "assigned_by_property"
+# Every condition that includes users pins a variant and matches only some users, so only those users enter the
+# experiment and everyone else gets no variant. Usually a test set-up, such as a list of the team's own users.
+CLOSED_TO_MATCHED_USERS = "closed_to_matched_users"
@frozen
@@ -108,6 +112,17 @@ def forced_variant_release_condition(ctx: HealthContext) -> CheckOutcome:
diagnostic_ref="A7b",
grade_evidence={"variant_property": variant_property.key, **set_after_launch.evidence},
)
+ if not any(condition.includes_everyone for condition in pins.pinned):
+ return _finding(
+ subcode=CLOSED_TO_MATCHED_USERS,
+ severity=ExperimentHealthFindingSeverity.WARNING,
+ title=f"Only users who match {set_names(pinned_numbers)} get a variant",
+ detail=_closed_detail(pinned_numbers, pinned_keys, is_draft, set_after_launch),
+ pinned_numbers=pinned_numbers,
+ pinned_keys=pinned_keys,
+ diagnostic_ref="A7b",
+ grade_evidence=set_after_launch.evidence,
+ )
return _finding(
subcode=ALL_CONDITIONS_PINNED,
severity=ExperimentHealthFindingSeverity.WARNING,
@@ -247,12 +262,13 @@ def _all_pinned_detail(
f"Every release condition that includes users sets a variant override ({_quoted(pinned_keys)}), so "
f"{when}each user gets the variant of the condition they match instead of a random one."
)
+ # The conditions can assign by segment on purpose, so the fix is offered for a random assignment.
why = (
"Each variant then holds a different group of users, so a difference in the results can come from "
"who those users are and not from the variant."
)
fix = (
- f"Remove {overrides} so users are assigned at random. Users who already saw a variant can then "
+ f"To assign users at random, remove {overrides}. Users who already saw a variant can then "
f"switch to another one. {set_after_launch.remedy_after_removal()}"
)
if is_draft:
@@ -260,6 +276,50 @@ def _all_pinned_detail(
return f"{fact}{set_after_launch.sentence} {why} {fix}"
+def _closed_detail(
+ numbers: Sequence[int], keys: Sequence[str], is_draft: bool, set_after_launch: _SetAfterLaunch
+) -> str:
+ enter = "after launch only those users enter" if is_draft else "only those users enter"
+ if len(keys) == 1:
+ gets = f'sets the variant override "{keys[0]}", so {enter} the experiment, and they all get "{keys[0]}".'
+ why = (
+ "Everyone else gets no variant. With no other variant to compare against, the results can't show "
+ f'whether "{keys[0]}" makes a difference.'
+ )
+ else:
+ gets = (
+ f"sets a variant override ({_quoted(keys)}), so {enter} the experiment, and each gets the variant of "
+ "their condition."
+ )
+ why = (
+ "Everyone else gets no variant. Each variant holds a different group of users, so a difference in the "
+ "results can come from who those users are and not from the variant."
+ )
+ if len(numbers) == 1:
+ fact = (
+ f"Set {numbers[0]} is the only release condition that includes users. It matches only some users and {gets}"
+ )
+ else:
+ fact = (
+ f"{set_names(numbers)} are the only release conditions that include users. Each matches only some users "
+ f"and {gets}"
+ )
+ overrides = "the override" if len(numbers) == 1 else "the overrides"
+ if is_draft:
+ fix = (
+ "An override helps to test the experiment before launch. Before you launch, add a release condition "
+ f"without a variant override for the users you want to test. If these are those users, remove {overrides} "
+ "instead."
+ )
+ else:
+ fix = (
+ "If these are test users, add a release condition without a variant override for the users you want to "
+ f"test, or end the experiment. If they are the users you want to test, remove {overrides} so they are "
+ f"assigned at random. {set_after_launch.remedy_after_removal()}"
+ )
+ return f"{fact}{set_after_launch.sentence} {why} {fix}"
+
+
def _assigned_by_property_detail(variant_property: str, set_after_launch: str) -> str:
return (
f'Each release condition that includes users matches the users whose property "{variant_property}" holds a '
diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.py
index d89c8c78c597..c8c1dd67e232 100644
--- a/products/experiments/backend/health/checks/zero_exposures.py
+++ b/products/experiments/backend/health/checks/zero_exposures.py
@@ -59,7 +59,10 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
# deletion.
if flag is None or flag.deleted:
return _cause_unknown(evidence, unit, ended=ctx.has_ended)
- if ctx.is_paused:
+ events = ctx.exposure_events
+ # A flag that no code called since launch exposes nobody after a resume either, so the missing calls are the cause.
+ never_called = events is not None and events.required_flag_key is not None and ctx.flag_called_during_run is False
+ if ctx.is_paused and not never_called:
# Info, because the flag-state check already reports the pause, and a pause needs no fix.
return _finding(
subcode=EXPERIMENT_PAUSED,
@@ -72,7 +75,6 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
evidence=evidence,
actions=(),
)
- events = ctx.exposure_events
if events is None:
return _cause_unknown(evidence, unit, ended=ctx.has_ended)
if events.required_flag_key is None:
@@ -132,6 +134,10 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:
"example with getFeatureFlag in the browser or with get_flag on the result of evaluate_flags in Python. "
f"{no_call_recorded} New calls can take up to {LAST_CALLED_AT_LAG} to show up here."
)
+ if ctx.is_paused:
+ detail += (
+ " The experiment is also paused, and resuming it exposes nobody until your code reads the flag."
+ )
# Info after the end, because no change to the code can expose more users then, so the finding only explains
# the empty results.
return _finding(
@@ -198,19 +204,31 @@ def _test_accounts_filtered_out(
) -> ExperimentHealthFinding:
exposed_units = f"the only {unit}" if exposed == 1 else f"all {exposed:,} {unit}s"
served = f'{exposed_units} that got a variant of the feature flag "{flag_key}"'
- fix = 'Turn off "Filter out internal and test users" in the exposure criteria'
+ turn_off = '"Filter out internal and test users" in the exposure criteria'
+ # When the filters match every user who got a variant, usually only the team's own users got one, and the filter
+ # then works as intended.
if ended:
title = f"No {unit}s exposed: the test account filter removed all of them"
detail = (
f"No {unit}s were exposed while the experiment ran, because the exposure criteria filter out test "
- f"accounts, and the project's test account filters matched {served}. {fix} to count them."
+ f"accounts, and the project's test account filters matched {served}. If these were your own team's "
+ f"{unit}s, the experiment didn't reach other {unit}s. Otherwise turn off {turn_off} to count them."
+ )
+ actions: tuple[ExperimentHealthFindingActionKind, ...] = (
+ ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,
)
else:
title = f"No {unit}s exposed: the test account filter removes all of them"
detail = (
f"The exposure criteria filter out test accounts, and the project's test account filters match {served} "
- f"during the experiment, so nobody is counted as exposed. {fix}, or change the project's test account "
- f"filters so that they don't match your {unit}s."
+ "during the experiment, so nobody is counted as exposed. If these are your own team's "
+ f"{unit}s, the experiment hasn't reached other {unit}s yet. The code that reads the flag may only run in "
+ "development, or the release conditions may include only your team. Otherwise change the project's test "
+ f"account filters so that they don't match your {unit}s, or turn off {turn_off}."
+ )
+ actions = (
+ ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG,
+ ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,
)
return _finding(
subcode=CALLS_NOT_COUNTED,
@@ -219,7 +237,7 @@ def _test_accounts_filtered_out(
title=title,
detail=detail,
evidence=evidence,
- actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,),
+ actions=actions,
)
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index 37b1cd7f61f2..eb3446afc093 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -180,6 +180,9 @@ class ReachableCondition:
variant_property: VariantProperty | None
# The cohorts whose members the condition matches. A `not_in` cohort filter matches the users outside the cohort.
member_cohort_ids: frozenset[int] = frozenset()
+ # Whether the condition has no filters, or only filters that leave users out, so it includes nearly everyone who
+ # reaches it.
+ includes_everyone: bool = False
@frozen
@@ -673,6 +676,30 @@ def _member_cohort_ids(condition: Mapping[str, Any]) -> frozenset[int]:
)
+_EXCLUDING_OPERATORS = frozenset(
+ {
+ PropertyOperator.IS_NOT,
+ PropertyOperator.NOT_ICONTAINS,
+ PropertyOperator.NOT_ICONTAINS_MULTI,
+ PropertyOperator.NOT_STARTS_WITH,
+ PropertyOperator.NOT_ENDS_WITH,
+ PropertyOperator.NOT_REGEX,
+ PropertyOperator.NOT_BETWEEN,
+ PropertyOperator.NOT_IN,
+ PropertyOperator.IS_NOT_SET,
+ PropertyOperator.SEMVER_NEQ,
+ PropertyOperator.IS_CLEANED_PATH_NOT_EXACT,
+ }
+)
+
+
+def _includes_everyone(condition: Mapping[str, Any]) -> bool:
+ properties = condition.get("properties")
+ if not isinstance(properties, list) or not properties:
+ return True
+ return all(isinstance(prop, dict) and prop.get("operator") in _EXCLUDING_OPERATORS for prop in properties)
+
+
def _reachable_condition(
number: int, condition: Mapping[str, Any], variant_keys: set[str], flag_aggregation: int | None
) -> ReachableCondition:
@@ -682,6 +709,7 @@ def _reachable_condition(
pinned_variant=pinned,
variant_property=None if pinned is None else _variant_property(condition, pinned, flag_aggregation),
member_cohort_ids=_member_cohort_ids(condition),
+ includes_everyone=_includes_everyone(condition),
)
diff --git a/products/experiments/backend/health/exposure_inputs.py b/products/experiments/backend/health/exposure_inputs.py
index 16aebe024c6e..d6a15e82d67b 100644
--- a/products/experiments/backend/health/exposure_inputs.py
+++ b/products/experiments/backend/health/exposure_inputs.py
@@ -13,6 +13,7 @@ from posthog.models.user import User
from products.experiments.backend.facade.contracts import HealthExposureInputsStatus
from products.experiments.backend.health.context import ExposureTotals
+from products.experiments.backend.health.launch_time import hours_since_launch
from products.experiments.backend.hogql_queries.experiment_exposures_query_runner import (
ExperimentExposuresQueryRunner,
build_experiment_exposure_query,
@@ -40,7 +41,9 @@ class ExposureInputs:
error: str | None
-def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_query_events: bool) -> ExposureInputs:
+def load_exposure_inputs(
+ experiment: Experiment, *, user: User | None, can_query_events: bool, can_read_flag_history: bool
+) -> ExposureInputs:
"""
Reads the cached answer of the experiment's exposure query. A missing or stale answer starts a calculation in the
background instead of blocking, because the query can run for minutes. Never raises.
@@ -95,7 +98,11 @@ def load_exposure_inputs(experiment: Experiment, *, user: User | None, can_query
sample_ratio_expected={},
daily_exposures={},
# The checks read the hours at computation, not now.
- hours_since_launch=(computed_at - experiment.start_date).total_seconds() / 3600 if computed_at else None,
+ hours_since_launch=(
+ hours_since_launch(experiment, computed_at, can_read_flag_history=can_read_flag_history)
+ if computed_at
+ else None
+ ),
counts_groups=aggregates_by_group(runner.group_type_index),
filters_test_accounts=applies_test_account_filters(experiment.team, query.exposure_criteria),
has_positive_test_account_filter=has_positive_test_account_filter(experiment.team),
diff --git a/products/experiments/backend/health/launch_time.py b/products/experiments/backend/health/launch_time.py
new file mode 100644
index 000000000000..d80ec7757de5
--- /dev/null
+++ b/products/experiments/backend/health/launch_time.py
@@ -0,0 +1,49 @@
+from datetime import datetime
+
+from posthog.exceptions_capture import capture_exception
+from posthog.models.activity_logging.activity_log import ActivityLog
+
+from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS
+from products.experiments.backend.models.experiment import Experiment
+
+
+def hours_since_launch(experiment: Experiment, at: datetime, *, can_read_flag_history: bool) -> float | None:
+ """
+ The hours from the experiment's launch to `at`, which the exposure checks wait on. The start date can be moved
+ back to before the launch, when the flag didn't serve variants yet, so the wait counts from the later of the start
+ date and the latest launch in the activity log. The activity log needs the access of the flag history, so a reader
+ without it waits from the start date.
+ """
+ start_date = experiment.start_date
+ if start_date is None:
+ return None
+ hours = (at - start_date).total_seconds() / 3600
+ # A wait that holds by the start date also holds by the launch, which can only be later.
+ if hours < ZERO_EXPOSURES_GRACE_HOURS or not can_read_flag_history:
+ return hours
+ launched_at = _latest_launch(experiment)
+ if launched_at is None or launched_at <= start_date:
+ return hours
+ return (at - launched_at).total_seconds() / 3600
+
+
+def _latest_launch(experiment: Experiment) -> datetime | None:
+ # A launch logs the start date that it sets on a draft. An experiment created with a start date logs none, and its
+ # start date stands.
+ try:
+ return (
+ ActivityLog.objects.filter(
+ team_id=experiment.team_id,
+ scope="Experiment",
+ item_id=str(experiment.id),
+ activity="updated",
+ detail__contains={"changes": [{"field": "start_date", "before": None}]},
+ )
+ .order_by("-created_at")
+ .values_list("created_at", flat=True)
+ .first()
+ )
+ except Exception as error:
+ # The start date is a safe fallback, so a failed lookup must not fail the exposure answer.
+ capture_exception(error)
+ return None
diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.py
index 7c5d4d8c0d7e..a285f9598cc4 100644
--- a/products/experiments/backend/health/loader.py
+++ b/products/experiments/backend/health/loader.py
@@ -47,7 +47,12 @@ def load_experiment_health_inputs(
experiment = _load_experiment(team_id, experiment_id)
return HealthInputs(
context=_health_context(experiment, reader_access, user),
- exposures=load_exposure_inputs(experiment, user=user, can_query_events=reader_access.can_query_events),
+ exposures=load_exposure_inputs(
+ experiment,
+ user=user,
+ can_query_events=reader_access.can_query_events,
+ can_read_flag_history=reader_access.can_read_flag_history,
+ ),
reader_access=reader_access,
flag_version=experiment.feature_flag.version if experiment.feature_flag_id else None,
project=build_project_setup_facts(
diff --git a/products/experiments/backend/health/test/test_bucketing_changed.py b/products/experiments/backend/health/test/test_bucketing_changed.py
index 49b9222f64a7..19cdf6144253 100644
--- a/products/experiments/backend/health/test/test_bucketing_changed.py
+++ b/products/experiments/backend/health/test/test_bucketing_changed.py
@@ -29,6 +29,7 @@ LAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)
FIRST_CHANGE_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
FIRST_CHANGE = "September 12, 2026 at 3:04 PM CEST"
SECOND_CHANGE = "September 12, 2026 at 3:21 PM CEST"
+THIRD_CHANGE = "September 12, 2026 at 3:38 PM CEST"
MIXED_ASSIGNMENTS = "A sample ratio mismatch or users in more than one variant can come from this change."
EVEN = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
KEY = BucketingSetting.KEY
@@ -134,9 +135,38 @@ class TestBucketingChangedMidRun(TestCase):
ExperimentHealthFindingSeverity.WARNING,
"The flag's variant assignment changed after launch",
),
- f'"Persist flag across authentication steps" was turned off on {FIRST_CHANGE}. The change came 3 '
- "hours after launch. Users who logged in got a new variant draw, so they may have switched variants. "
- f"{MIXED_ASSIGNMENTS} Reset the experiment, or move its start date to {FIRST_CHANGE}.",
+ f'On {FIRST_CHANGE}, "Persist flag across authentication steps" was turned off. The change came 3 '
+ "hours after launch. Users who logged in while it was on no longer keep the variant they got before "
+ f"logging in, so they may have switched variants. {MIXED_ASSIGNMENTS} Reset the experiment, or move "
+ f"its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "experience_continuity_turned_on_after_a_set_back_change",
+ _context(
+ (IDENTIFIER, "distinct_id", "device_id"),
+ (IDENTIFIER, "device_id", "distinct_id"),
+ (CONTINUITY, False, True),
+ ),
+ (
+ "experience_continuity_changed",
+ ExperimentHealthFindingSeverity.WARNING,
+ "The flag's variant assignment changed after launch",
+ ),
+ f'On {THIRD_CHANGE}, "Persist flag across authentication steps" was turned on. Users exposed between '
+ "these changes may have switched variants. A sample ratio mismatch or users in more than one variant "
+ f"can come from these changes. Reset the experiment, or move its start date to {THIRD_CHANGE}.",
+ ),
+ (
+ "experience_continuity_turned_on",
+ _context((CONTINUITY, False, True), exposed_units=5),
+ (
+ "experience_continuity_changed",
+ ExperimentHealthFindingSeverity.INFO,
+ "The flag's variant assignment changed after launch",
+ ),
+ f'On {FIRST_CHANGE}, "Persist flag across authentication steps" was turned on. Turning it on '
+ "doesn't move users who were already exposed to another variant, unless they log in later on another "
+ "device or after logging out.",
),
(
"aggregation_changed",
@@ -154,8 +184,8 @@ class TestBucketingChangedMidRun(TestCase):
"two_settings_name_the_key_first",
_context((CONTINUITY, True, False), (KEY, "old-key", "new-key")),
("key_changed", ExperimentHealthFindingSeverity.WARNING, "The flag key changed after launch"),
- f'The flag key changed from "old-key" to "new-key" on {SECOND_CHANGE}. "Persist flag across '
- f'authentication steps" was turned off on {FIRST_CHANGE}. The key is an input of the variant draw, so '
+ f'The flag key changed from "old-key" to "new-key" on {SECOND_CHANGE}. On {FIRST_CHANGE}, "Persist '
+ 'flag across authentication steps" was turned off. The key is an input of the variant draw, so '
'every user got a new one. Exposures under "old-key" are no longer in the results, and code that '
'still uses "old-key" no longer gets this flag. Reset the experiment, or move its start date to '
f"{SECOND_CHANGE}.",
diff --git a/products/experiments/backend/health/test/test_exposure_inputs.py b/products/experiments/backend/health/test/test_exposure_inputs.py
index 388252197d89..1c1a679f6a75 100644
--- a/products/experiments/backend/health/test/test_exposure_inputs.py
+++ b/products/experiments/backend/health/test/test_exposure_inputs.py
@@ -108,7 +108,7 @@ class TestLoadExposureInputs(BaseTest):
"products.experiments.backend.health.exposure_inputs.ExperimentExposuresQueryRunner.run",
return_value=response,
):
- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)
+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)
self.assertEqual(
(inputs.status, inputs.error), (expected_status, CALCULATION_ERROR if refresh_failed else None)
@@ -144,7 +144,7 @@ class TestLoadExposureInputs(BaseTest):
exposure_criteria={"multiple_variant_handling": "not_a_handling"},
)
- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)
+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)
self.assertEqual(inputs.status, HealthExposureInputsStatus.FAILED)
@@ -152,7 +152,7 @@ class TestLoadExposureInputs(BaseTest):
experiment = self._create_experiment("exposure-inputs-failed-calculation")
cache_key = self._store_failed_calculation(experiment)
- inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True)
+ inputs = load_exposure_inputs(experiment, user=self.user, can_query_events=True, can_read_flag_history=True)
self.assertEqual((inputs.status, inputs.error), (HealthExposureInputsStatus.FAILED, CALCULATION_ERROR))
self.assertEqual(get_query_status(team_id=self.team.pk, query_id=cache_key).error_message, CALCULATION_ERROR)
diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.py
index 3185c8502024..deda497d2d00 100644
--- a/products/experiments/backend/health/test/test_forced_variant.py
+++ b/products/experiments/backend/health/test/test_forced_variant.py
@@ -43,6 +43,14 @@ IOS: dict[str, Any] = {
"properties": [{"key": "$device_type", "value": "iOS", "operator": "exact", "type": "person"}],
"rollout_percentage": 100,
}
+NOT_STAFF: dict[str, Any] = {
+ "properties": [{"key": "email", "value": "@example.com", "operator": "not_icontains", "type": "person"}],
+ "rollout_percentage": 100,
+}
+NOT_IOS: dict[str, Any] = {
+ "properties": [{"key": "$device_type", "value": "iOS", "operator": "is_not", "type": "person"}],
+ "rollout_percentage": 100,
+}
CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
@@ -128,6 +136,16 @@ class TestForcedVariantReleaseCondition(TestCase):
(
"cohorts_forced_into_both_variants",
_context([{**IOS, "variant": "test"}, {**STAFF, "variant": "control"}]),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test, control", None),
+ ),
+ (
+ "everyone_but_staff_forced_into_one_variant",
+ _context([{**NOT_STAFF, "variant": "test"}]),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "test", None),
+ ),
+ (
+ "segment_and_its_complement_forced_into_both_variants",
+ _context([{**IOS, "variant": "test"}, {**NOT_IOS, "variant": "control"}]),
("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test, control", None),
),
(
@@ -148,7 +166,7 @@ class TestForcedVariantReleaseCondition(TestCase):
{**_property_is("plan", "pro"), "variant": "test"},
]
),
- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
),
(
"variants_mapped_from_a_person_and_a_group_property_of_one_key",
@@ -158,12 +176,27 @@ class TestForcedVariantReleaseCondition(TestCase):
{**_property_is("arm", "test", group_type_index=0), "variant": "test"},
]
),
- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
),
(
"one_variant_mapped_from_a_property",
_context([{**_property_is("arm", "control"), "variant": "control"}]),
- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1", "control", None),
+ ),
+ (
+ "list_is_the_only_condition_with_users",
+ _context([{**STAFF, "variant": "test"}, {**EVERYONE, "rollout_percentage": 0}]),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1", "test", None),
+ ),
+ (
+ "draft_list_is_the_only_condition_with_users",
+ _context([{**STAFF, "variant": "test"}, {**IOS, "variant": "test"}], status=DRAFT),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test", None),
+ ),
+ (
+ "list_and_catch_all_pin_one_variant",
+ _context([{**STAFF, "variant": "test"}, {**EVERYONE, "variant": "test"}]),
+ ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "test", None),
),
(
"variants_mapped_from_two_properties",
@@ -173,7 +206,7 @@ class TestForcedVariantReleaseCondition(TestCase):
{**_property_is("cell", "test"), "variant": "test"},
]
),
- ("all_conditions_pinned", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
+ ("closed_to_matched_users", ExperimentHealthFindingSeverity.WARNING, "1, 2", "control, test", None),
),
("override_after_a_catch_all_is_unreachable", _context([EVERYONE, {**STAFF, "variant": "test"}]), None),
(
@@ -406,6 +439,54 @@ class TestForcedVariantReleaseCondition(TestCase):
)
)
+ @parameterized.expand(
+ [
+ (
+ "one_list",
+ [{**STAFF, "variant": "test"}, {**EVERYONE, "rollout_percentage": 0}],
+ "Only users who match Set 1 get a variant",
+ "Set 1 is the only release condition that includes users. It matches only some users and sets the "
+ 'variant override "test", so only those users enter the experiment, and they all get "test". Everyone '
+ "else gets no variant. With no other variant to compare against, the results can't show whether "
+ '"test" makes a difference. If these are test users, add a release condition without a variant '
+ "override for the users you want to test, or end the experiment. If they are the users you want to "
+ "test, remove the override so they are assigned at random. Then reset the experiment, or move its "
+ "start date to after the removal.",
+ ),
+ (
+ "lists_for_both_variants",
+ [{**STAFF, "variant": "test"}, {**IOS, "variant": "control"}],
+ "Only users who match Sets 1 and 2 get a variant",
+ "Sets 1 and 2 are the only release conditions that include users. Each matches only some users and "
+ 'sets a variant override ("test", "control"), so only those users enter the experiment, and each gets '
+ "the variant of their condition. Everyone else gets no variant. Each variant holds a different group "
+ "of users, so a difference in the results can come from who those users are and not from the "
+ "variant. If these are test users, add a release condition without a variant override for the users "
+ "you want to test, or end the experiment. If they are the users you want to test, remove the "
+ "overrides so they are assigned at random. Then reset the experiment, or move its start date to after "
+ "the removal.",
+ ),
+ (
+ "segment_and_its_complement",
+ [{**IOS, "variant": "test"}, {**NOT_IOS, "variant": "control"}],
+ "No users are randomly assigned to a variant",
+ 'Every release condition that includes users sets a variant override ("test", "control"), so each '
+ "user gets the variant of the condition they match instead of a random one. Each variant then holds a "
+ "different group of users, so a difference in the results can come from who those users are and not "
+ "from the variant. To assign users at random, remove the overrides. Users who already saw a variant "
+ "can then switch to another one. Then reset the experiment, or move its start date to after the "
+ "removal.",
+ ),
+ ]
+ )
+ def test_names_who_can_enter_when_every_condition_pins(
+ self, _name: str, groups: list[dict[str, Any]], title: str, detail: str
+ ) -> None:
+ finding = forced_variant_release_condition(_context(groups))
+
+ assert isinstance(finding, ExperimentHealthFinding)
+ self.assertEqual((finding.title, finding.detail), (title, detail))
+
def test_variants_mapped_from_a_property_say_who_assigns_them(self) -> None:
finding = forced_variant_release_condition(
_context(
diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.py
index 33a766c8f645..163a1b4c6f79 100644
--- a/products/experiments/backend/health/test/test_zero_exposures.py
+++ b/products/experiments/backend/health/test/test_zero_exposures.py
@@ -92,11 +92,25 @@ class TestZeroExposures(TestCase):
"flag_called_by_test_accounts_only",
{"flag_called_during_run": True, "exposures": TEST_ACCOUNTS_ONLY},
CALLS_NOT_COUNTED,
- (Action.EDIT_EXPOSURE_CRITERIA,),
+ (Action.OPEN_FEATURE_FLAG, Action.EDIT_EXPOSURE_CRITERIA),
WARNING,
),
("calls_unknown", {"flag_called_during_run": None}, None, (Action.EDIT_EXPOSURE_CRITERIA,), WARNING),
("paused", {"is_paused": True, "flag_called_during_run": True}, EXPERIMENT_PAUSED, (), INFO),
+ (
+ "paused_flag_not_called",
+ {"is_paused": True, "flag_called_during_run": False},
+ FLAG_NOT_CALLED,
+ (),
+ WARNING,
+ ),
+ (
+ "paused_custom_event",
+ {"is_paused": True, "exposure_events": CUSTOM_EVENT, "flag_called_during_run": False},
+ EXPERIMENT_PAUSED,
+ (),
+ INFO,
+ ),
(
"custom_event_whatever_the_flag_calls",
{"exposure_events": CUSTOM_EVENT, "flag_called_during_run": False},
@@ -173,6 +187,10 @@ class TestZeroExposures(TestCase):
assert isinstance(finding, ExperimentHealthFinding)
self.assertEqual((finding.subcode, finding.actions, finding.severity), (subcode, actions, severity))
self.assertEqual(finding.detail.startswith("No users were exposed while the experiment ran"), ctx.has_ended)
+ self.assertEqual(
+ finding.detail.endswith("resuming it exposes nobody until your code reads the flag."),
+ ctx.is_paused and subcode == FLAG_NOT_CALLED,
+ )
@parameterized.expand(
[
@@ -221,8 +239,11 @@ class TestZeroExposures(TestCase):
"No users exposed: the test account filter removes all of them",
"The exposure criteria filter out test accounts, and the project's test account filters match all 1,200 "
'users that got a variant of the feature flag "checkout-test" during the experiment, so nobody is '
- 'counted as exposed. Turn off "Filter out internal and test users" in the exposure criteria, or change '
- "the project's test account filters so that they don't match your users.",
+ "counted as exposed. If these are your own team's users, the experiment hasn't reached other users "
+ "yet. The code that reads the flag may only run in development, or the release conditions may include "
+ "only your team. Otherwise change the project's test account filters so "
+ 'that they don\'t match your users, or turn off "Filter out internal and test users" in the exposure '
+ "criteria.",
),
(
"ended_one_user",
@@ -233,8 +254,9 @@ class TestZeroExposures(TestCase):
"No users exposed: the test account filter removed all of them",
"No users were exposed while the experiment ran, because the exposure criteria filter out test "
"accounts, and the project's test account filters matched the only user that got a variant of the "
- 'feature flag "checkout-test". Turn off "Filter out internal and test users" in the exposure criteria '
- "to count them.",
+ "feature flag \"checkout-test\". If these were your own team's users, the experiment didn't reach "
+ 'other users. Otherwise turn off "Filter out internal and test users" in the exposure criteria to '
+ "count them.",
),
(
"nobody_without_the_filter",
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index ba8031444be7..6814d8dcf7b8 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -73,6 +73,7 @@ from products.experiments.backend.health.context import (
load_health_context,
)
from products.experiments.backend.health.exposure_gate import load_split_history
+from products.experiments.backend.health.launch_time import hours_since_launch
from products.experiments.backend.health.registry import (
EXPOSURE_HEALTH_CHECKS,
evaluate as evaluate_health,
@@ -582,7 +583,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
sample_ratio_mismatch_p_value=sample_ratio_mismatch.p_value if sample_ratio_mismatch else None,
sample_ratio_expected=sample_ratio_mismatch.expected if sample_ratio_mismatch else {},
daily_exposures=daily_exposures,
- hours_since_launch=self.experiment.hours_since_launch,
+ hours_since_launch=hours_since_launch(
+ self.experiment, datetime.now(UTC), can_read_flag_history=self._can_read_flag_history()
+ ),
counts_groups=aggregates_by_group(self.group_type_index),
filters_test_accounts=applies_test_account_filters(self.team, self.exposure_criteria),
has_positive_test_account_filter=has_positive_test_account_filter(self.team),
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index 6ff0cc4fe9bf..721c59d115cf 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -1,4 +1,5 @@
import re
+from contextlib import nullcontext
from datetime import UTC, datetime, timedelta
from typing import Any
from zoneinfo import ZoneInfo
@@ -7,6 +8,7 @@ import time_machine
from posthog.test.base import _create_event, _create_person, flush_persons_and_events, snapshot_clickhouse_queries
from unittest.mock import patch
+from django.db import OperationalError
from django.forms.models import model_to_dict
from django.test import override_settings
@@ -2000,19 +2002,46 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
@parameterized.expand(
[
- ("first_day", "2024-01-01T23:00:00Z", None, []),
+ ("first_day", "2024-01-01T23:00:00Z", None, None, []),
(
"after_a_day",
"2024-01-02T01:00:00Z",
None,
+ None,
[(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],
),
(
"after_a_day_with_flag_calls",
"2024-01-02T01:00:00Z",
datetime(2024, 1, 1, 12, tzinfo=UTC),
+ None,
[(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "calls_not_counted")],
),
+ ("first_day_after_a_launch_with_an_earlier_start_date", "2024-01-02T01:00:00Z", None, 12, []),
+ (
+ "after_a_day_from_a_launch_with_an_earlier_start_date",
+ "2024-01-02T13:00:00Z",
+ None,
+ 12,
+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],
+ ),
+ (
+ "launch_hidden_from_a_reader_without_the_flag_history",
+ "2024-01-02T01:00:00Z",
+ None,
+ 12,
+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],
+ False,
+ ),
+ (
+ "launch_lookup_fails",
+ "2024-01-02T01:00:00Z",
+ None,
+ 12,
+ [(ExperimentExposureHealthFindingCode.ZERO_EXPOSURES, "flag_not_called")],
+ True,
+ True,
+ ),
]
)
def test_zero_exposures_finding_waits_a_day_and_names_the_flag_calls(
@@ -2020,10 +2049,23 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
_name: str,
now: str,
last_called_at: datetime | None,
+ launched_hours_after_start: int | None,
expected: list[tuple[ExperimentExposureHealthFindingCode, str]],
+ reads_flag_history: bool = True,
+ launch_lookup_fails: bool = False,
) -> None:
self.feature_flag.last_called_at = last_called_at
self.feature_flag.save()
+ if launched_hours_after_start is not None:
+ launched_at = datetime(2024, 1, 1, tzinfo=UTC) + timedelta(hours=launched_hours_after_start)
+ ActivityLog.objects.create(
+ team_id=self.team.id,
+ scope="Experiment",
+ item_id=str(self.experiment.id),
+ activity="updated",
+ detail={"changes": [{"field": "start_date", "before": None, "after": launched_at.isoformat()}]},
+ created_at=launched_at,
+ )
query = ExperimentExposureQuery(
kind="ExperimentExposureQuery",
experiment_id=self.experiment.id,
@@ -2034,8 +2076,14 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
exposure_criteria=self.experiment.exposure_criteria,
)
- with time_machine.travel(now, tick=False):
- response = ExperimentExposuresQueryRunner(team=self.team, query=query).calculate()
+ lookup = patch(
+ "products.experiments.backend.health.launch_time.ActivityLog.objects.filter",
+ side_effect=OperationalError("the activity log is unavailable"),
+ )
+ with time_machine.travel(now, tick=False), lookup if launch_lookup_fails else nullcontext():
+ response = ExperimentExposuresQueryRunner(
+ team=self.team, query=query, user=self.user if reads_flag_history else None
+ ).calculate()
self.assertEqual(response.total_exposures, {"control": 0, "test": 0})
self.assertEqual([(finding.code, finding.subcode) for finding in response.health_findings or []], expected)
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index ad935fe6ac68..f7eb63cd6900 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -1057,7 +1057,10 @@ class EnterpriseExperimentsViewSet(
`running_but_no_rollout` when 10 or more people got a variant from flag calls without a
flag version that are no local evaluation, from an hour after the window starts. When
every condition that includes users pins the variant whose key one property holds,
- `assigned_by_property` replaces `all_conditions_pinned`.
+ `assigned_by_property` replaces `all_conditions_pinned`. When none of them includes
+ everyone, so that every other user gets no variant, `closed_to_matched_users` replaces it,
+ as a warning. A condition without filters, or with only filters that leave users out,
+ includes everyone.
A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The record
is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index c51e981ffff3..4f494329bc92 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -813,7 +813,10 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* `running_but_no_rollout` when 10 or more people got a variant from flag calls without a
* flag version that are no local evaluation, from an hour after the window starts. When
* every condition that includes users pins the variant whose key one property holds,
- * `assigned_by_property` replaces `all_conditions_pinned`.
+ * `assigned_by_property` replaces `all_conditions_pinned`. When none of them includes
+ * everyone, so that every other user gets no variant, `closed_to_matched_users` replaces it,
+ * as a warning. A condition without filters, or with only filters that leave users out,
+ * includes everyone.
*
* A draft whose flag has no recorded call gets the info `new_flag_no_traffic`. The record
* is the flag's `last_called_at`, which a periodic task updates, so it trails a call. The