Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): note a flag change before 5 exposures as info (#120997)

Marcel Poelker18fc3cd9786d
diff --git a/products/experiments/backend/health/checks/bucketing_changed.py b/products/experiments/backend/health/checks/bucketing_changed.py
index ef13611c8b30..f93ac065d319 100644
--- a/products/experiments/backend/health/checks/bucketing_changed.py
+++ b/products/experiments/backend/health/checks/bucketing_changed.py
@@ -3,7 +3,8 @@ A flag setting that decides the variant draw changed after launch while people w
group type the flag matches by, the bucketing identifier, or persistence across authentication. Pure functions, no
I/O: `health/exposure_gate.py` loads the changes and the exposure answers.
-People exposed before such a change get a new draw, so they can get another variant after it.
+People exposed before such a change get a new draw, so they can get another variant after it. When fewer than
+MIN_EXPOSED_UNITS_TO_WARN were exposed before the last change, that can't move the results, so it is info.
"""
from products.experiments.backend.facade.contracts import (
@@ -23,7 +24,10 @@ from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
changes_not_checked,
delay_after_launch,
+ exposed_when,
+ few_exposed_units,
history_to_check,
+ leave_out,
not_checked_finding,
)
from products.experiments.backend.health.context import (
@@ -58,6 +62,7 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | N
if effect is None:
return None
+ few = few_exposed_units(history, ExperimentHealthFlagChangeKind.BUCKETING)
if effect.shape == BucketingChangeShape.REVERTED:
first_at = format_change_time(effect.first.changed_at, history.team_timezone)
changed = _changed_settings(effect)
@@ -74,13 +79,22 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | N
],
)
last_at = format_change_time(effect.last.changed_at, history.team_timezone)
+ set_back = f"{changed} changed on {first_at} and {_was(effect)} set back on {last_at}."
+ if few is not None:
+ return _finding(
+ history,
+ effect,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title=_TITLE,
+ sentences=[set_back, exposed_when(history, effect, few), *leave_out(history, effect, few)],
+ )
return _finding(
history,
effect,
severity=ExperimentHealthFindingSeverity.WARNING,
title=_TITLE,
sentences=[
- f"{changed} changed on {first_at} and {_was(effect)} set back on {last_at}.",
+ set_back,
*delay_after_launch(history, effect),
"People exposed in between got a new variant draw, so they may have switched variants.",
f"Move the start date to {suggested_start(effect.last.changed_at, history.team_timezone)}, or "
@@ -88,14 +102,29 @@ def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | N
],
)
+ title = _KEY_TITLE if effect.shape == BucketingChangeShape.KEY_CHANGED else _TITLE
+ what_changed = [_what_changed(net, history.team_timezone) for net in effect.net_changes]
+ if few is not None:
+ return _finding(
+ history,
+ effect,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title=title,
+ sentences=[
+ *what_changed,
+ exposed_when(history, effect, few),
+ *leave_out(history, effect, few),
+ *_key_state(effect),
+ ],
+ )
start_at = suggested_start(effect.last.changed_at, history.team_timezone)
return _finding(
history,
effect,
severity=ExperimentHealthFindingSeverity.WARNING,
- title=_KEY_TITLE if effect.shape == BucketingChangeShape.KEY_CHANGED else _TITLE,
+ title=title,
sentences=[
- *(_what_changed(net, history.team_timezone) for net in effect.net_changes),
+ *what_changed,
*delay_after_launch(history, effect),
*_consequence(effect),
f"Reset the experiment, or move its start date to {start_at}.",
@@ -167,6 +196,15 @@ def _consequence(effect: BucketingNetEffect) -> list[str]:
]
+def _key_state(effect: BucketingNetEffect) -> list[str]:
+ if effect.shape != BucketingChangeShape.KEY_CHANGED:
+ return []
+ key = next(change for change in effect.net_changes if change.setting == BucketingSetting.KEY)
+ return [
+ f"Code that still uses {describe_bucketing_value(BucketingSetting.KEY, key.before)} no longer gets this flag."
+ ]
+
+
def _changed_settings(effect: BucketingNetEffect) -> str:
names = list(dict.fromkeys(_SETTING_NAMES[setting] for setting in effect.changed_settings))
joined = names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"
@@ -188,6 +226,7 @@ def _finding(
return change_finding(
history,
effect,
+ kind=ExperimentHealthFlagChangeKind.BUCKETING,
code=ExperimentHealthFindingCode.BUCKETING_CHANGED_MID_RUN,
subcode=effect.shape.value,
severity=severity,
diff --git a/products/experiments/backend/health/checks/conditions_changed.py b/products/experiments/backend/health/checks/conditions_changed.py
index 410f4bc11ee6..58e961eb2e58 100644
--- a/products/experiments/backend/health/checks/conditions_changed.py
+++ b/products/experiments/backend/health/checks/conditions_changed.py
@@ -9,8 +9,9 @@ How many is unknown, so that is info too.
A variant override that was removed moves the people its condition matches from its variant to a random one, and
their exposures under that variant stay in the results. It is graded by the share of that variant's users the
-condition serves, as the forced-variant check grades an override in place. An override that is still in place is
-the forced-variant check's to report, so it is not reported here.
+condition serves, as the forced-variant check grades an override in place. When fewer than MIN_EXPOSED_UNITS_TO_WARN
+were exposed before the last change, a warning by that grade is info, because so few can't move the results. An
+override that is still in place is the forced-variant check's to report, so it is not reported here.
"""
from products.experiments.backend.facade.contracts import (
@@ -23,7 +24,10 @@ from products.experiments.backend.health.checks.change_wording import suggested_
from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
changes_not_checked,
+ exposed_when,
+ few_exposed_units,
history_to_check,
+ leave_out,
not_checked_finding,
)
from products.experiments.backend.health.condition_changes import (
@@ -186,17 +190,38 @@ def _override_removed(
if pins is None or not pins.unweighted:
return None
severity = ExperimentHealthFindingSeverity.INFO
+ subcode = (
+ ConditionsChangeShape.REVERTED
+ if effect.shape == ConditionsChangeShape.REVERTED
+ else ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED
+ )
+ what_changed = [_removed(override, history.team_timezone) for override in removed]
+ few = few_exposed_units(history, ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS)
+ if severity == ExperimentHealthFindingSeverity.WARNING and few is not None:
+ return _finding(
+ history,
+ effect,
+ subcode=subcode,
+ title=_OVERRIDE_TITLE,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ sentences=[
+ *what_changed,
+ exposed_when(history, effect, few),
+ *leave_out(history, effect, few),
+ *_measured(grade, removed),
+ ],
+ grade=grade,
+ diagnostic_ref="A7",
+ )
last_changed_at = max(override.restored_at or override.removed_at for override in removed)
return _finding(
history,
effect,
- subcode=ConditionsChangeShape.REVERTED
- if effect.shape == ConditionsChangeShape.REVERTED
- else ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ subcode=subcode,
title=_OVERRIDE_TITLE,
severity=severity,
sentences=[
- *(_removed(override, history.team_timezone) for override in removed),
+ *what_changed,
*_moved(removed),
*_measured(grade, removed),
f"Reset the experiment, or move its start date to "
@@ -292,6 +317,7 @@ def _finding(
return change_finding(
history,
effect,
+ kind=ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS,
code=ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN,
subcode=subcode.value,
severity=severity,
diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.py
index 7b74441db67e..c68f063a88a0 100644
--- a/products/experiments/backend/health/checks/flag_change_findings.py
+++ b/products/experiments/backend/health/checks/flag_change_findings.py
@@ -12,14 +12,18 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthFlagChangeKind,
)
from products.experiments.backend.health.checks.change_wording import (
delay,
+ duration,
not_checked_because,
not_checked_next_step,
+ suggested_start,
the_change,
)
from products.experiments.backend.health.context import (
+ MIN_EXPOSED_UNITS_TO_WARN,
FlagChange,
FlagHistory,
FlagHistoryNotChecked,
@@ -143,6 +147,7 @@ def change_finding(
history: FlagHistory,
effect: ChangeEffect,
*,
+ kind: ExperimentHealthFlagChangeKind,
code: ExperimentHealthFindingCode,
subcode: str,
severity: ExperimentHealthFindingSeverity,
@@ -157,14 +162,47 @@ def change_finding(
severity=severity,
title=title,
detail=" ".join(sentences),
- evidence=change_evidence(
- history, effect, kind_evidence=kind_evidence, exposed_before_change=history.exposed_before_first
- ),
+ evidence={
+ **change_evidence(
+ history, effect, kind_evidence=kind_evidence, exposed_before_change=history.exposed_before_first
+ ),
+ # pinned: capped at MIN_EXPOSED_UNITS_TO_WARN, so the cap means "at least". None when unknown.
+ "units_exposed_before_last_change": history.exposed_units_before_last.get(kind),
+ },
actions=(ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),
diagnostic_ref=diagnostic_ref,
)
+def few_exposed_units(history: FlagHistory, kind: ExperimentHealthFlagChangeKind) -> int | None:
+ """The persons or groups exposed before the last counted change of a kind, when they are too few for a warning.
+ None when there were enough, or when the count is unknown, which keeps the warning."""
+ count = history.exposed_units_before_last.get(kind)
+ return count if count is not None and count < MIN_EXPOSED_UNITS_TO_WARN else None
+
+
+def exposed_when(history: FlagHistory, effect: ChangeEffect, count: int) -> str:
+ after_launch = duration((effect.last.changed_at - history.launched_at).total_seconds())
+ change = "The change" if effect.change_count == 1 else "The last change"
+ return f"{change} came {after_launch} after launch, when {_units(count, history.counts_groups)} had been exposed."
+
+
+def leave_out(history: FlagHistory, effect: ChangeEffect, count: int) -> list[str]:
+ """The next step for the few persons or groups exposed before the last change. Empty when there were none."""
+ if count == 0:
+ return []
+ start_at = suggested_start(effect.last.changed_at, history.team_timezone)
+ return [f"Move the start date to {start_at} so the results count only exposures after it."]
+
+
+def _units(count: int, counts_groups: bool) -> str:
+ if count == 0:
+ return "no group" if counts_groups else "nobody"
+ if counts_groups:
+ return "1 group" if count == 1 else f"{count} groups"
+ return "1 person" if count == 1 else f"{count} people"
+
+
def not_checked_finding(
history: FlagHistoryNotChecked,
effect: ChangeEffect,
diff --git a/products/experiments/backend/health/checks/split_changed.py b/products/experiments/backend/health/checks/split_changed.py
index bba42f2ab64b..8d60158c2d6c 100644
--- a/products/experiments/backend/health/checks/split_changed.py
+++ b/products/experiments/backend/health/checks/split_changed.py
@@ -2,7 +2,9 @@
A variant split of the experiment's flag that changed after launch while people were already exposed. Pure
functions, no I/O: `health/exposure_gate.py` loads the changes and the exposure answers.
-People exposed before such a change can get another variant after it, so the results mix two assignments.
+People exposed before such a change can get another variant after it, so the results mix two assignments. When
+fewer than MIN_EXPOSED_UNITS_TO_WARN were exposed before the last change, that can't move the results, so it is info.
+A variant rolled out to 100% stays a warning, because the split serves one variant from then on.
"""
from typing import Literal
@@ -18,7 +20,10 @@ from products.experiments.backend.health.checks.flag_change_findings import (
change_finding,
changes_not_checked,
delay_after_launch,
+ exposed_when,
+ few_exposed_units,
history_to_check,
+ leave_out,
not_checked_finding,
)
from products.experiments.backend.health.checks.flag_state import is_single_variant_shipped
@@ -68,6 +73,11 @@ def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
title=_REVERTED_INFO_TITLE,
sentences=[*_reverted(history, effect), "Nobody was exposed in between."],
)
+ few = few_exposed_units(history, ExperimentHealthFlagChangeKind.SPLIT)
+ # A variant at 100% serves everyone the same variant from then on, so the experiment compares nothing, however
+ # few people were exposed before.
+ if few is not None and effect.shape != SplitChangeShape.VARIANT_ROLLED_OUT:
+ return _few_exposed(history, effect, few)
if effect.shape == SplitChangeShape.REVERTED:
start_at = suggested_start(effect.last.changed_at, history.team_timezone)
return _finding(
@@ -126,6 +136,28 @@ def _not_checked(history: FlagHistoryNotChecked, flag: FlagState) -> ExperimentH
)
+def _few_exposed(history: FlagHistory, effect: SplitNetEffect, count: int) -> ExperimentHealthFinding:
+ what_changed = (
+ _reverted(history, effect) if effect.shape == SplitChangeShape.REVERTED else [_what_changed(history, effect)]
+ )
+ from_then_on: list[str] = []
+ if effect.shape == SplitChangeShape.VARIANT_ZEROED:
+ keys = _quoted(zeroed_variant_keys(effect.from_split, effect.to_split), conjunction="and")
+ from_then_on = [f"No new users get {keys}."]
+ return _finding(
+ history,
+ effect,
+ severity=ExperimentHealthFindingSeverity.INFO,
+ title=_TITLE,
+ sentences=[
+ *what_changed,
+ exposed_when(history, effect, count),
+ *leave_out(history, effect, count),
+ *from_then_on,
+ ],
+ )
+
+
def _reported_as_shipped(effect: SplitNetEffect, flag: FlagState) -> bool:
# `variant_shipped_while_running` reports this state, and its detail carries the date of the rollout.
return effect.shape == SplitChangeShape.VARIANT_ROLLED_OUT and is_single_variant_shipped(flag)
@@ -234,6 +266,7 @@ def _finding(
return change_finding(
history,
effect,
+ kind=ExperimentHealthFlagChangeKind.SPLIT,
code=ExperimentHealthFindingCode.SPLIT_CHANGED_MID_RUN,
subcode=effect.shape.value,
severity=severity,
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index e3a22b8d5035..aa537c7612e7 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -123,6 +123,12 @@ class FlagHistory:
# Whether anyone was exposed while an override was set and removed again, or removed and set back. A user
# exposed only outside that window got the same variant throughout. None when unknown.
exposed_during_overrides: Mapping[ChangeWindow, bool | None]
+ # Whether the experiment counts groups rather than persons, by the rule the exposure query uses.
+ counts_groups: bool
+ # The persons or groups exposed between launch and the last counted change of a kind, up to
+ # MIN_EXPOSED_UNITS_TO_WARN. Asked only for a kind with a counted change, so a kind that was not asked is absent.
+ # None when unknown.
+ exposed_units_before_last: Mapping[ExperimentHealthFlagChangeKind, int | None]
def of_kind[C: FlagChange](self, kind: type[C]) -> tuple[C, ...]:
return changes_of_kind(self.changes, kind)
@@ -147,6 +153,8 @@ class FlagHistoryNotCheckedReason(StrEnum):
MAX_FLAG_SAVES = 100
MAX_RELEASE_CONDITIONS = 100
+# A few early exposures, usually the creator's own test, can't move a result, so a change before this many is info.
+MIN_EXPOSED_UNITS_TO_WARN = 5
@frozen
diff --git a/products/experiments/backend/health/exposure_gate.py b/products/experiments/backend/health/exposure_gate.py
index 2b62f12ae581..f147c2b4d07f 100644
--- a/products/experiments/backend/health/exposure_gate.py
+++ b/products/experiments/backend/health/exposure_gate.py
@@ -1,7 +1,8 @@
"""
Loads the changes of a running experiment's flag since launch that can move exposed people: the split, the
settings that decide the variant draw, and the release conditions. Then asks ClickHouse whether anyone was exposed
-before each change. A change before the first exposure moved nobody, so it does not count.
+before each change. A change before the first exposure moved nobody, so it does not count. For each kind of change
+that counts, it also counts the persons or groups exposed before the last change of that kind, up to a few.
The answer for a past window does not change, so it is cached per window and shared by every reader. A slow
answer is then paid once per change, not on every load of the experiment page. A failed query is cached as
@@ -10,12 +11,15 @@ unknown too, so a scan that is too large for a request runs at most once a day.
Only exposure criteria that name an event are asked about. The answer for a property filter or an action can
depend on what each reader may read, and can change as the filter matches other events over time, so no cached
answer for them is safe to share. The changes of those experiments are reported as not checked.
+
+The answers run without the reader and are shared, so a reader denied any event property gets none. Every change
+then counts for that reader, as if every answer were unknown.
"""
import json
import hashlib
from bisect import bisect_left
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from time import monotonic
@@ -63,6 +67,7 @@ from products.experiments.backend.health.condition_changes import (
from products.experiments.backend.health.context import (
MAX_FLAG_SAVES,
MAX_RELEASE_CONDITIONS,
+ MIN_EXPOSED_UNITS_TO_WARN,
BucketingChange,
BucketingSetting,
ChangeWindow,
@@ -82,7 +87,9 @@ from products.experiments.backend.health.split_changes import (
split_variant_keys,
)
from products.experiments.backend.hogql_queries.exposure_query_logic import (
+ aggregates_by_group,
build_exposure_event_conditions,
+ get_entity_key,
get_exposure_event_and_property,
normalize_to_exposure_criteria,
resolve_default_exposure_event,
@@ -102,6 +109,14 @@ _ANSWER_CACHE_SECONDS = int(timedelta(days=30).total_seconds())
_TOO_EXPENSIVE_CACHE_SECONDS = int(timedelta(days=1).total_seconds())
_FAILED_CACHE_SECONDS = int(timedelta(hours=1).total_seconds())
UNKNOWN = "unknown"
+# The two questions have their own key prefixes, so a cached bool of "anyone exposed" is never read as a count.
+_EXPOSED_KEY_PREFIX = "experiment_split_exposure"
+_EXPOSED_UNITS_KEY_PREFIX = "experiment_exposed_units"
+_KINDS: dict[ExperimentHealthFlagChangeKind, type[FlagChange]] = {
+ ExperimentHealthFlagChangeKind.SPLIT: SplitChange,
+ ExperimentHealthFlagChangeKind.BUCKETING: BucketingChange,
+ ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS: ConditionsChange,
+}
# The activity log fields of a flag save that can move exposed people.
_FILTERS = "filters"
_SETTING_FIELDS = {
@@ -179,6 +194,75 @@ def query_any_exposure(
"variant_keys": ast.Constant(value=list(variant_keys)),
},
)
+ return bool(_execute(experiment, query))
+
+
+def query_exposed_units(
+ experiment: Experiment,
+ *,
+ window_start: datetime,
+ window_end: datetime,
+ served: Sequence[tuple[str, Sequence[str]]],
+ group_type_index: int | None,
+ default_exposure_event: str,
+) -> int:
+ """
+ The persons or groups exposed in `[window_start, window_end)`, up to MIN_EXPOSED_UNITS_TO_WARN. `served` pairs each
+ flag key with the variant keys the flag served under it, and an exposure counts when it matches one of the pairs.
+ """
+ team = experiment.team
+ # `person_id` joins an aggregation of the team's person overrides, which runs in full whatever the LIMIT. The raw
+ # id on the event needs no join. A person merged after an exposure can then count twice, which only keeps a warning.
+ unit = get_entity_key(group_type_index) if aggregates_by_group(group_type_index) else "event_person_id"
+ conditions: list[ast.Expr] = []
+ for flag_key, variant_keys in served:
+ _, variant_property = get_exposure_event_and_property(
+ flag_key, experiment.exposure_criteria, default_exposure_event=default_exposure_event
+ )
+ conditions.append(
+ ast.And(
+ exprs=[
+ *build_exposure_event_conditions(
+ experiment.exposure_criteria, team, flag_key, default_exposure_event=default_exposure_event
+ ),
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.In,
+ left=ast.Field(chain=["properties", variant_property]),
+ right=ast.Constant(value=list(variant_keys)),
+ ),
+ ]
+ )
+ )
+ # The exposure query leaves out group exposures without a group key, so they don't count here either.
+ has_unit = (
+ ast.Call(name="notEmpty", args=[ast.Field(chain=[unit])])
+ if aggregates_by_group(group_type_index)
+ else ast.Constant(value=True)
+ )
+ # No test-account filter, as in `query_any_exposure`. DISTINCT with LIMIT stops the scan at the cap.
+ query = parse_select(
+ """
+ SELECT count() FROM (
+ SELECT DISTINCT {unit} FROM events
+ WHERE timestamp >= {window_start} AND timestamp < {window_end} AND {served} AND {has_unit}
+ LIMIT {cap}
+ )
+ """,
+ placeholders={
+ "unit": ast.Field(chain=[unit]),
+ "window_start": ast.Constant(value=window_start),
+ "window_end": ast.Constant(value=window_end),
+ "served": ast.Or(exprs=conditions),
+ "has_unit": has_unit,
+ "cap": ast.Constant(value=MIN_EXPOSED_UNITS_TO_WARN),
+ },
+ )
+ results = _execute(experiment, query)
+ return int(results[0][0]) if results else 0
+
+
+def _execute(experiment: Experiment, query: ast.SelectQuery | ast.SelectSetQuery) -> list[Any]:
+ team = experiment.team
with tags_context(
product=Product.EXPERIMENTS, feature=Feature.HEALTH_CHECK, team_id=team.pk, org_id=team.organization_id
):
@@ -191,7 +275,7 @@ def query_any_exposure(
max_execution_time=EXPOSURE_QUERY_TIMEOUT_SECONDS, timeout_overflow_mode="throw"
),
)
- return bool(response.results)
+ return response.results or []
@frozen
@@ -203,30 +287,105 @@ class _Window:
variant_keys: tuple[str, ...]
+@frozen
+class _UnitsWindow:
+ start: datetime
+ end: datetime
+ # Each flag key the flag had in the window, with the variant keys it served under that key.
+ served: tuple[tuple[str, tuple[str, ...]], ...]
+
+
class _ExposureGate:
"""
Answers the exposure questions of one request from the cache, or with at most MAX_EXPOSURE_QUERIES queries
that start within REQUEST_BUDGET_SECONDS.
"""
- def __init__(self, experiment: Experiment, launched_at: datetime, *, prefetch: Sequence[_Window]) -> None:
+ def __init__(
+ self,
+ experiment: Experiment,
+ launched_at: datetime,
+ *,
+ group_type_index: int | None,
+ denied_event_properties: frozenset[str],
+ prefetch: Sequence[_Window | _UnitsWindow],
+ ) -> None:
self._experiment = experiment
self._launched_at = launched_at
+ self._group_type_index = group_type_index
+ self._denied_event_properties = denied_event_properties
self._default_exposure_event = resolve_default_exposure_event(experiment.team, launched_at)
- # One cache read for all windows of the walk, so a flag with many changes costs one round trip.
- self._prefetched_keys = {self._cache_key(window) for window in prefetch if window.variant_keys}
- self._prefetched = cache.get_many(list(self._prefetched_keys))
+ # One cache read for all windows of the walk and the counts, so a flag with many changes costs one round trip.
+ self._prefetched_keys = (
+ set()
+ if denied_event_properties
+ else {
+ self._units_cache_key(window) if isinstance(window, _UnitsWindow) else self._cache_key(window)
+ for window in prefetch
+ if isinstance(window, _UnitsWindow) or window.variant_keys
+ }
+ )
+ self._prefetched = cache.get_many(list(self._prefetched_keys)) if self._prefetched_keys else {}
self._started_at = monotonic()
self._query_count = 0
self._stopped = False
def exposed(self, window: _Window) -> bool | None:
- """None when unknown: the query failed in this request or within its cache time, or the request is out of budget."""
+ """None when unknown: the query failed in this request or within its cache time, the request is out of budget,
+ or the reader is denied an event property."""
if not window.variant_keys:
return False
- cache_key = self._cache_key(window)
+ return self._answer(
+ self._cache_key(window),
+ bool,
+ lambda: query_any_exposure(
+ self._experiment,
+ flag_key=window.flag_key,
+ window_start=window.start,
+ window_end=window.end,
+ variant_keys=window.variant_keys,
+ default_exposure_event=self._default_exposure_event,
+ ),
+ window_end=window.end,
+ final=lambda answer: answer,
+ )
+
+ def exposed_units(self, window: _UnitsWindow | None) -> int | None:
+ """The persons or groups exposed in the window, up to MIN_EXPOSED_UNITS_TO_WARN. None when unknown, as for
+ `exposed`, or without a window, because the flag served no variants for a while."""
+ if window is None:
+ return None
+ return self._answer(
+ self._units_cache_key(window),
+ int,
+ lambda: query_exposed_units(
+ self._experiment,
+ window_start=window.start,
+ window_end=window.end,
+ served=window.served,
+ group_type_index=self._group_type_index,
+ default_exposure_event=self._default_exposure_event,
+ ),
+ window_end=window.end,
+ # Late events can only add units, so a count at the cap can't change.
+ final=lambda count: count >= MIN_EXPOSED_UNITS_TO_WARN,
+ )
+
+ def _answer[A: (bool, int)](
+ self,
+ cache_key: str,
+ answer_type: type[A],
+ ask: Callable[[], A],
+ *,
+ window_end: datetime,
+ final: Callable[[A], bool],
+ ) -> A | None:
+ # Every answer reads the flag, variant and group properties of the exposures. Any denied event property, not
+ # a list of the read ones, so no list can miss one. The findings then keep their warnings.
+ if self._denied_event_properties:
+ return None
cached = self._prefetched.get(cache_key) if cache_key in self._prefetched_keys else cache.get(cache_key)
- if isinstance(cached, bool):
+ if isinstance(cached, answer_type):
return cached
if cached == UNKNOWN:
return None
@@ -238,43 +397,48 @@ class _ExposureGate:
return None
self._query_count += 1
try:
- answer = query_any_exposure(
- self._experiment,
- flag_key=window.flag_key,
- window_start=window.start,
- window_end=window.end,
- variant_keys=window.variant_keys,
- default_exposure_event=self._default_exposure_event,
- )
+ answer = ask()
except Exception as error:
cache_failed_scan(cache_key, error)
self._stopped = True
return None
- settled = answer or timezone.now() - window.end > _SETTLED_AFTER
+ settled = final(answer) or timezone.now() - window_end > _SETTLED_AFTER
cache.set(cache_key, answer, _ANSWER_CACHE_SECONDS if settled else _RECENT_NOBODY_CACHE_SECONDS)
# Two kinds can ask the same window, and a prefetched miss would otherwise send the second to ClickHouse.
self._prefetched[cache_key] = answer
return answer
def _cache_key(self, window: _Window) -> str:
+ return self._key(
+ _EXPOSED_KEY_PREFIX,
+ window.start.isoformat(),
+ window.end.isoformat(),
+ window.flag_key,
+ sorted(window.variant_keys),
+ )
+
+ def _units_cache_key(self, window: _UnitsWindow) -> str:
+ # The cap is an input, so a count cached under another cap is never read as one under this cap.
+ return self._key(
+ _EXPOSED_UNITS_KEY_PREFIX,
+ window.start.isoformat(),
+ window.end.isoformat(),
+ [[flag_key, sorted(variant_keys)] for flag_key, variant_keys in window.served],
+ self._group_type_index,
+ MIN_EXPOSED_UNITS_TO_WARN,
+ )
+
+ def _key(self, prefix: str, *window_inputs: object) -> str:
experiment = self._experiment
# Every input of the query. The resolved exposure event is one, because the exposure event rollout
# can move an experiment to another event.
inputs = json.dumps(
- [
- self._launched_at.isoformat(),
- window.start.isoformat(),
- window.end.isoformat(),
- window.flag_key,
- sorted(window.variant_keys),
- experiment.exposure_criteria,
- self._default_exposure_event,
- ],
+ [self._launched_at.isoformat(), *window_inputs, experiment.exposure_criteria, self._default_exposure_event],
sort_keys=True,
default=str,
)
digest = hashlib.sha256(inputs.encode()).hexdigest()
- return f"experiment_split_exposure:{experiment.team_id}:{experiment.id}:{digest}"
+ return f"{prefix}:{experiment.team_id}:{experiment.id}:{digest}"
class _ConditionAggregationMismatch(Exception):
@@ -343,6 +507,8 @@ def _load_flag_history(
variant_keys=split_variant_keys(parse_variants(flag.variants)),
),
exposed_during_overrides={},
+ counts_groups=aggregates_by_group(flag.aggregation_group_type_index),
+ exposed_units_before_last={},
)
times = _change_times(changes)
if not times:
@@ -360,7 +526,20 @@ def _load_flag_history(
)
starts = (launched_at, *times[:-1])
walk = [timeline.window(start=start, end=end) for start, end in zip(starts, times)]
- gate = _ExposureGate(experiment, launched_at, prefetch=walk)
+ # The last change of a kind is its last counted change whenever the kind has one, so these windows are known
+ # before the walk and share its cache read.
+ units_windows = {
+ kind: timeline.units_window(start=launched_at, end=of_kind[-1].changed_at)
+ for kind, change_type in _KINDS.items()
+ if (of_kind := changes_of_kind(changes, change_type))
+ }
+ gate = _ExposureGate(
+ experiment,
+ launched_at,
+ group_type_index=flag.aggregation_group_type_index,
+ denied_event_properties=reader_access.denied_event_properties,
+ prefetch=[*walk, *(window for window in units_windows.values() if window is not None)],
+ )
for window in walk:
exposed_before = gate.exposed(window)
if exposed_before is not False:
@@ -368,8 +547,15 @@ def _load_flag_history(
break
if history.counted_since is None:
return history
+ # Asked before the set-back and override questions, because a count can lower the finding of every kind.
+ exposed_units_before_last = {
+ kind: gate.exposed_units(units_windows[kind])
+ for kind, change_type in _KINDS.items()
+ if history.counted(change_type)
+ }
return replace(
history,
+ exposed_units_before_last=exposed_units_before_last,
exposed_between=_exposed_between_set_backs(gate, timeline, history),
exposed_during_overrides=_exposed_during_overrides(gate, timeline, history),
)
@@ -431,24 +617,38 @@ class _Timeline:
index = bisect_left(self._split_times, at)
return self._split_changes[index].before if index < len(self._split_changes) else self._current_split
+ def units_window(self, *, start: datetime, end: datetime) -> _UnitsWindow | None:
+ """The window up to a change, with every key and split that the flag served in it. None when the flag served
+ no variants for a while."""
+ served = self.served_between(start=start, end=end)
+ return None if served is None else _UnitsWindow(start=start, end=end, served=served)
+
+ def served_between(self, *, start: datetime, end: datetime) -> tuple[tuple[str, tuple[str, ...]], ...] | None:
+ """The variant keys that the flag served in `[start, end)`, by flag key. None when it served a flag without
+ variants for a while."""
+ variant_keys_by_flag_key: dict[str, set[str]] = {}
+ for at in self.points_within(start=start, end=end):
+ split = self.split_before(at)
+ # A flag without variants serves `true`, so people can be exposed with no variant key to look for.
+ if not split_variant_keys(split):
+ return None
+ variant_keys_by_flag_key.setdefault(self.key_before(at), set()).update(split_variant_keys(split))
+ return tuple(
+ (flag_key, tuple(sorted(variant_keys))) for flag_key, variant_keys in variant_keys_by_flag_key.items()
+ )
+
def points_within(self, *, start: datetime, end: datetime) -> list[datetime]:
return [at for at in self._times if start < at <= end]
def _exposed_between(gate: _ExposureGate, timeline: _Timeline, *, start: datetime, end: datetime) -> bool | None:
"""Whether anyone was exposed in `[start, end)`, under any key and split that the flag served then."""
- variant_keys_by_flag_key: dict[str, set[str]] = {}
- for at in timeline.points_within(start=start, end=end):
- split = timeline.split_before(at)
- # A flag without variants serves `true`, so people can be exposed with no variant key to look for.
- if not split_variant_keys(split):
- return None
- variant_keys_by_flag_key.setdefault(timeline.key_before(at), set()).update(split_variant_keys(split))
+ served = timeline.served_between(start=start, end=end)
+ if served is None:
+ return None
answers = []
- for flag_key, variant_keys in variant_keys_by_flag_key.items():
- answer = gate.exposed(
- _Window(start=start, end=end, flag_key=flag_key, variant_keys=tuple(sorted(variant_keys)))
- )
+ for flag_key, variant_keys in served:
+ answer = gate.exposed(_Window(start=start, end=end, flag_key=flag_key, variant_keys=variant_keys))
if answer:
return True
answers.append(answer)
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index 9f31012e9574..7bddcc2fa2e4 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -60,6 +60,8 @@ def flag_history(
not_checked: FlagHistoryNotCheckedReason | None = None,
release_conditions: tuple[ReleaseCondition, ...] | None = None,
exposed_during_overrides: Mapping[ChangeWindow, bool | None] | None = None,
+ exposed_units_before_last: Mapping[ExperimentHealthFlagChangeKind, int | None] | None = None,
+ counts_groups: bool = False,
) -> FlagHistory | FlagHistoryNotChecked:
# Every change counts from the first, as when someone was exposed before it.
if not_checked is not None:
@@ -77,4 +79,6 @@ def flag_history(
if release_conditions is not None
else next((change.after for change in reversed(changes) if isinstance(change, ConditionsChange)), ()),
exposed_during_overrides=exposed_during_overrides or {},
+ counts_groups=counts_groups,
+ exposed_units_before_last=exposed_units_before_last or {},
)
diff --git a/products/experiments/backend/health/test/test_bucketing_changed.py b/products/experiments/backend/health/test/test_bucketing_changed.py
index 4e693184d767..ed4e20c12d87 100644
--- a/products/experiments/backend/health/test/test_bucketing_changed.py
+++ b/products/experiments/backend/health/test/test_bucketing_changed.py
@@ -44,6 +44,7 @@ def _context(
is_running: bool = True,
variants: list[dict[str, Any]] = EVEN,
not_checked: FlagHistoryNotCheckedReason | None = None,
+ exposed_units: int | None = None,
) -> HealthContext:
bucketing = [
BucketingChange(
@@ -64,6 +65,7 @@ def _context(
exposed_before_first=exposed_before_first,
exposed_between={ExperimentHealthFlagChangeKind.BUCKETING: exposed_between},
not_checked=not_checked,
+ exposed_units_before_last={ExperimentHealthFlagChangeKind.BUCKETING: exposed_units},
),
)
@@ -80,6 +82,34 @@ class TestBucketingChangedMidRun(TestCase):
'and code that still uses "old-key" no longer gets this flag. Reset the experiment, or move its start '
f"date to {FIRST_CHANGE}.",
),
+ (
+ "key_changed_before_4_people",
+ _context((KEY, "old-key", "new-key"), exposed_units=4),
+ ("key_changed", ExperimentHealthFindingSeverity.INFO, "The flag key changed after launch"),
+ f'The flag key changed from "old-key" to "new-key" on {FIRST_CHANGE}. The change came 3 hours after '
+ f"launch, when 4 people had been exposed. Move the start date to {FIRST_CHANGE} so the results count only exposures after it. "
+ 'Code that still uses "old-key" no longer gets this flag.',
+ ),
+ (
+ "key_changed_before_5_people",
+ _context((KEY, "old-key", "new-key"), exposed_units=5),
+ ("key_changed", ExperimentHealthFindingSeverity.WARNING, "The flag key changed after launch"),
+ f'The flag key changed from "old-key" to "new-key" on {FIRST_CHANGE}. The key is an input of the '
+ 'variant draw, so every user got a new one. Exposures under "old-key" are no longer in the results, '
+ 'and code that still uses "old-key" no longer gets this flag. Reset the experiment, or move its start '
+ f"date to {FIRST_CHANGE}.",
+ ),
+ (
+ "aggregation_changed_before_4_people",
+ _context((AGGREGATION, None, 0), exposed_units=4),
+ (
+ "aggregation_changed",
+ ExperimentHealthFindingSeverity.INFO,
+ "The flag's variant assignment changed after launch",
+ ),
+ f"The flag changed from matching by users to matching by groups on {FIRST_CHANGE}. The change came 3 "
+ f"hours after launch, when 4 people had been exposed. Move the start date to {FIRST_CHANGE} so the results count only exposures after it.",
+ ),
(
"bucketing_identifier_changed_days_after_launch",
_context((IDENTIFIER, "distinct_id", "device_id"), first_change_at=FIRST_CHANGE_AT + timedelta(days=8)),
@@ -143,6 +173,23 @@ class TestBucketingChangedMidRun(TestCase):
"People exposed in between got a new variant draw, so they may have switched variants. Move the "
f"start date to {SECOND_CHANGE}, or reset the experiment.",
),
+ (
+ "set_back_before_4_people",
+ _context(
+ (IDENTIFIER, "distinct_id", "device_id"),
+ (IDENTIFIER, "device_id", "distinct_id"),
+ exposed_between=True,
+ exposed_units=4,
+ ),
+ (
+ "reverted",
+ ExperimentHealthFindingSeverity.INFO,
+ "The flag's variant assignment changed after launch",
+ ),
+ f'The flag\'s "Match by" setting changed on {FIRST_CHANGE} and was set back on {SECOND_CHANGE}. '
+ "The last change came 3 hours after launch, when 4 people had been exposed. Move the start date to "
+ f"{SECOND_CHANGE} so the results count only exposures after it.",
+ ),
(
"set_back_with_nobody_in_between",
_context((KEY, "old-key", "new-key"), (KEY, "new-key", "old-key"), exposed_between=False),
diff --git a/products/experiments/backend/health/test/test_conditions_changed.py b/products/experiments/backend/health/test/test_conditions_changed.py
index 82786861d0c5..f51a1a3e2dfd 100644
--- a/products/experiments/backend/health/test/test_conditions_changed.py
+++ b/products/experiments/backend/health/test/test_conditions_changed.py
@@ -54,6 +54,7 @@ def _context(
now: list[dict[str, Any]] | None = None,
exposed_during_overrides: dict[ChangeWindow, bool | None] | None = None,
variants: list[dict[str, Any]] = EVEN,
+ exposed_units: int | None = None,
) -> HealthContext:
variant_keys = [variant["key"] for variant in variants]
changes = [
@@ -74,6 +75,7 @@ def _context(
not_checked=not_checked,
release_conditions=None if now is None else parse_release_conditions(now, variant_keys=variant_keys),
exposed_during_overrides=exposed_during_overrides,
+ exposed_units_before_last={ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS: exposed_units},
),
condition_set_traffic=None if people_by_set is None else _traffic(people_by_set),
)
@@ -129,6 +131,31 @@ class TestReleaseConditionsChangedMidRun(TestCase):
f'{TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" with the override in '
f"place (300 of about 1,650). Reset the experiment, or move its start date to {FIRST_CHANGE}.",
),
+ (
+ "override_removed_from_a_large_share_before_4_people",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 300, 2: 2700}, exposed_units=4),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. The change came 3 hours after '
+ f"launch, when 4 people had been exposed. Move the start date to {FIRST_CHANGE} so the results count only exposures after it. "
+ f'Since {TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" with the '
+ "override in place (300 of about 1,650).",
+ ),
+ (
+ "override_removed_from_a_large_share_before_5_people",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 300, 2: 2700}, exposed_units=5),
+ ("variant_override_changed", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} Since '
+ f'{TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" with the override in '
+ f"place (300 of about 1,650). Reset the experiment, or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_without_a_measurement_before_4_people",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], exposed_units=4),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '
+ "can't measure how many users Set 1 matches. Reset the experiment, or move its start date to "
+ f"{FIRST_CHANGE}.",
+ ),
(
"override_removed_from_a_few_percent",
_context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 30, 2: 2970}),
diff --git a/products/experiments/backend/health/test/test_exposure_gate.py b/products/experiments/backend/health/test/test_exposure_gate.py
index bec74a414bcd..dccc38e0fd31 100644
--- a/products/experiments/backend/health/test/test_exposure_gate.py
+++ b/products/experiments/backend/health/test/test_exposure_gate.py
@@ -15,9 +15,11 @@ from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded, ClickHouseQ
from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.team.team import Team
from posthog.models.user import User
+from posthog.test.persons import create_person
from products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind, ExperimentHealthReaderAccess
from products.experiments.backend.health.context import (
+ MIN_EXPOSED_UNITS_TO_WARN,
BucketingChange,
BucketingSetting,
ChangeWindow,
@@ -32,6 +34,7 @@ from products.experiments.backend.health.exposure_gate import (
MAX_OVERRIDE_WINDOWS,
load_flag_history,
query_any_exposure,
+ query_exposed_units,
)
from products.experiments.backend.hogql_queries.exposure_query_logic import DEFAULT_EXPOSURE_EVENT
from products.experiments.backend.models.experiment import Experiment
@@ -128,12 +131,19 @@ class TestLoadFlagHistory(BaseTest):
self,
answers: list[bool | Exception] | Callable[..., bool],
*,
+ units: list[int | Exception] | Callable[..., int] | None = None,
reader_access: ExperimentHealthReaderAccess = FULL_ACCESS,
) -> tuple[FlagHistory | FlagHistoryNotChecked | None, MagicMock]:
- with patch(
- "products.experiments.backend.health.exposure_gate.query_any_exposure", side_effect=answers
- ) as query:
+ with (
+ patch("products.experiments.backend.health.exposure_gate.query_any_exposure", side_effect=answers) as query,
+ patch(
+ "products.experiments.backend.health.exposure_gate.query_exposed_units",
+ side_effect=units,
+ return_value=MIN_EXPOSED_UNITS_TO_WARN,
+ ) as units_query,
+ ):
history = load_flag_history(self.experiment, reader_access=reader_access)
+ self.units_query = units_query
return history, query
def test_changes_before_the_first_exposure_do_not_count(self) -> None:
@@ -207,6 +217,78 @@ class TestLoadFlagHistory(BaseTest):
(query.call_args.kwargs["flag_key"], query.call_args.kwargs["window_start"]), ("renamed", changed_at[0])
)
+ def test_counts_the_units_exposed_before_the_last_counted_change_of_each_kind(self) -> None:
+ first_at = self.launched_at + timedelta(days=1)
+ changed_at = [first_at, first_at + timedelta(hours=1), first_at + timedelta(hours=2)]
+ self._log_change(changed_at[0], key=("first-key", FLAG_KEY))
+ self._log_split_changes([_split(50, 50), _split(60, 40), _split(70, 30)], first_at=changed_at[1])
+
+ history, _ = self._load(
+ [True], units=lambda *_args, **kwargs: 2 if kwargs["window_end"] == changed_at[0] else 3
+ )
+
+ assert isinstance(history, FlagHistory)
+ self.assertEqual(
+ history.exposed_units_before_last,
+ {ExperimentHealthFlagChangeKind.SPLIT: 3, ExperimentHealthFlagChangeKind.BUCKETING: 2},
+ )
+ self.assertEqual(
+ [
+ (call.kwargs["window_start"], call.kwargs["window_end"], call.kwargs["served"])
+ for call in self.units_query.call_args_list
+ ],
+ [
+ (
+ self.launched_at,
+ changed_at[2],
+ (("first-key", ("control", "test")), (FLAG_KEY, ("control", "test"))),
+ ),
+ (self.launched_at, changed_at[0], (("first-key", ("control", "test")),)),
+ ],
+ )
+
+ def test_a_count_is_cached_apart_from_the_exposure_answer_of_the_same_window(self) -> None:
+ self._log_split_changes([_split(50, 50), _split(60, 40)])
+ first, _ = self._load([True], units=[3])
+
+ with patch("products.experiments.backend.health.exposure_gate.cache", wraps=cache) as cache_spy:
+ second, query = self._load([])
+
+ assert isinstance(first, FlagHistory) and isinstance(second, FlagHistory)
+ self.assertEqual(first.exposed_units_before_last, {ExperimentHealthFlagChangeKind.SPLIT: 3})
+ self.assertEqual(second.exposed_units_before_last, {ExperimentHealthFlagChangeKind.SPLIT: 3})
+ self.assertEqual((query.call_count, self.units_query.call_count), (0, 0))
+ self.assertEqual((cache_spy.get_many.call_count, cache_spy.get.call_count), (1, 0))
+
+ @parameterized.expand(
+ [
+ ("variant_property", frozenset({"$feature_flag_response"}), (0, None, None)),
+ ("group_key", frozenset({"$group_0"}), (0, None, None)),
+ ("unrelated_property", frozenset({"$browser"}), (0, None, None)),
+ ("no_denied_property", frozenset(), (1, True, 3)),
+ ]
+ )
+ def test_a_reader_denied_any_event_property_gets_no_exposure_answer(
+ self, _name: str, denied: frozenset[str], expected: tuple[int, bool | None, int | None]
+ ) -> None:
+ changed_at = self._log_split_changes([_split(50, 50), _split(60, 40), _split(70, 30)])
+ self._load([False, True], units=[3])
+
+ history, query = self._load(
+ [],
+ reader_access=ExperimentHealthReaderAccess(
+ can_read_flag_history=True, can_query_events=True, denied_event_properties=denied
+ ),
+ )
+
+ assert isinstance(history, FlagHistory)
+ counted_from, exposed_before_first, units = expected
+ self.assertEqual(
+ (history.counted_since, history.exposed_before_first, history.exposed_units_before_last),
+ (changed_at[counted_from], exposed_before_first, {ExperimentHealthFlagChangeKind.SPLIT: units}),
+ )
+ self.assertEqual((query.call_count, self.units_query.call_count), (0, 0))
+
def test_no_exposure_before_any_change(self) -> None:
self._log_split_changes([_split(50, 50), _split(60, 40), _split(70, 30)])
@@ -466,6 +548,8 @@ class TestLoadFlagHistory(BaseTest):
self.assertEqual(query.call_count, expected_queries)
self.assertEqual(history.counted(SplitChange)[0].changed_at, changed_at[expected_queries])
self.assertIsNone(history.exposed_before_first)
+ self.assertEqual(history.exposed_units_before_last, {ExperimentHealthFlagChangeKind.SPLIT: None})
+ self.assertEqual(self.units_query.call_count, 0)
def test_cached_walk_reads_the_cache_once(self) -> None:
self._log_split_changes([_split(50, 50), _split(60, 40)] * 5)
@@ -481,27 +565,31 @@ class TestLoadFlagHistory(BaseTest):
@parameterized.expand(
[
- ("exposed", True, timedelta(minutes=10), timedelta(days=2), 0),
- ("nobody_in_a_settled_window", False, timedelta(hours=2), timedelta(days=2), 0),
- ("nobody_in_a_recent_window_minutes_later", False, timedelta(minutes=10), timedelta(minutes=1), 0),
- ("nobody_in_a_recent_window_later", False, timedelta(minutes=10), timedelta(minutes=10), 1),
+ ("exposed", True, 5, timedelta(minutes=10), timedelta(days=2), 0, 0),
+ ("nobody_in_a_settled_window", False, None, timedelta(hours=2), timedelta(days=2), 0, 0),
+ ("nobody_in_a_recent_window_minutes_later", False, None, timedelta(minutes=10), timedelta(minutes=1), 0, 0),
+ ("nobody_in_a_recent_window_later", False, None, timedelta(minutes=10), timedelta(minutes=10), 1, 0),
+ ("few_units_in_a_settled_window", True, 3, timedelta(hours=2), timedelta(days=2), 0, 0),
+ ("few_units_in_a_recent_window_later", True, 3, timedelta(minutes=10), timedelta(minutes=10), 0, 1),
]
)
def test_answer_cache(
self,
_name: str,
answer: bool,
+ units: int | None,
change_age: timedelta,
next_load_after: timedelta,
expected_queries: int,
+ expected_units_queries: int,
) -> None:
self._log_split_changes([_split(50, 50), _split(60, 40)], first_at=timezone.now() - change_age)
- self._load([answer])
+ self._load([answer], units=None if units is None else [units])
with time_machine.travel(timezone.now() + next_load_after):
- _, query = self._load([answer])
+ _, query = self._load([answer], units=None if units is None else [units])
- self.assertEqual(query.call_count, expected_queries)
+ self.assertEqual((query.call_count, self.units_query.call_count), (expected_queries, expected_units_queries))
def test_revert_through_a_flag_without_variants_leaves_in_between_unknown(self) -> None:
self._log_split_changes([_split(50, 50), [], _split(50, 50)])
@@ -572,3 +660,68 @@ class TestQueryAnyExposure(ClickhouseTestMixin, BaseTest):
)
self.assertIs(exposed, expected)
+
+
+class TestQueryExposedUnits(ClickhouseTestMixin, BaseTest):
+ @parameterized.expand(
+ [
+ ("capped", None, [(f"person-{index}", FLAG_KEY, "test", None) for index in range(7)], 5),
+ (
+ "each_person_once_under_every_key",
+ None,
+ [
+ ("person-1", "old-key", "control", None),
+ ("person-1", FLAG_KEY, "control", None),
+ ("person-2", FLAG_KEY, "test", None),
+ ("person-3", FLAG_KEY, "blue", None),
+ ],
+ 2,
+ ),
+ (
+ "groups_with_a_group_key",
+ 0,
+ [
+ ("person-1", FLAG_KEY, "test", "org-1"),
+ ("person-2", FLAG_KEY, "control", "org-1"),
+ ("person-3", FLAG_KEY, "test", "org-2"),
+ ("person-4", FLAG_KEY, "test", None),
+ ],
+ 2,
+ ),
+ ]
+ )
+ def test_query_exposed_units(
+ self,
+ _name: str,
+ group_type_index: int | None,
+ exposures: list[tuple[str, str, str, str | None]],
+ expected: int,
+ ) -> None:
+ now = timezone.now()
+ experiment = _create_running_experiment(self.team, self.user, launched_at=now - timedelta(days=5))
+ for distinct_id in dict.fromkeys(distinct_id for distinct_id, *_ in exposures):
+ create_person(team=self.team, distinct_ids=[distinct_id])
+ for distinct_id, flag_key, variant, group_key in exposures:
+ _create_event(
+ team=self.team,
+ event="$feature_flag_called",
+ distinct_id=distinct_id,
+ properties={
+ "$feature_flag": flag_key,
+ "$feature_flag_response": variant,
+ **({"$group_0": group_key} if group_key else {}),
+ },
+ timestamp=now - timedelta(days=3),
+ )
+ flush_persons_and_events()
+
+ units = query_exposed_units(
+ experiment,
+ window_start=now - timedelta(days=5),
+ window_end=now - timedelta(days=1),
+ served=[("old-key", ["control", "test"]), (FLAG_KEY, ["control", "test"])],
+ group_type_index=group_type_index,
+ default_exposure_event=DEFAULT_EXPOSURE_EVENT,
+ )
+
+ self.assertEqual(units, expected)
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index d90475a50a12..9c2ac56f7eeb 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -58,6 +58,8 @@ def _context(
archived: bool = False,
deleted: bool = False,
not_checked: FlagHistoryNotCheckedReason | None = None,
+ exposed_units: int | None = None,
+ counts_groups: bool = False,
) -> HealthContext:
changes = [
SplitChange(
@@ -78,6 +80,8 @@ def _context(
exposed_before_first=exposed_before_first,
exposed_between={ExperimentHealthFlagChangeKind.SPLIT: exposed_between},
not_checked=not_checked,
+ exposed_units_before_last={ExperimentHealthFlagChangeKind.SPLIT: exposed_units},
+ counts_groups=counts_groups,
),
)
@@ -118,6 +122,47 @@ class TestSplitChangedMidRun(TestCase):
f"Reset the experiment, or move its start date to {FIRST_CHANGE} to analyze only the people exposed "
f"after the change. {MIXED_ASSIGNMENTS}",
),
+ (
+ "rebalanced_before_4_people",
+ _context(EVEN, UNEVEN, exposed_units=4),
+ ("rebalanced", ExperimentHealthFindingSeverity.INFO, "E3"),
+ f"The split changed from control 50% / test 50% to control 80% / test 20% on {FIRST_CHANGE}. "
+ "The change came 3 hours after launch, when 4 people had been exposed. Move the start date to "
+ f"{FIRST_CHANGE} so the results count only exposures after it.",
+ ),
+ (
+ "rebalanced_before_5_people",
+ _context(EVEN, UNEVEN, exposed_units=5),
+ ("rebalanced", ExperimentHealthFindingSeverity.WARNING, "E3"),
+ f"The split changed from control 50% / test 50% to control 80% / test 20% on {FIRST_CHANGE}. "
+ "People exposed before the change may have switched variants, so the results mix two assignments. "
+ f"Reset the experiment, or move its start date to {FIRST_CHANGE} to analyze only the people exposed "
+ f"after the change. {MIXED_ASSIGNMENTS}",
+ ),
+ (
+ "rebalanced_before_any_group",
+ _context(EVEN, UNEVEN, exposed_units=0, counts_groups=True),
+ ("rebalanced", ExperimentHealthFindingSeverity.INFO, "E3"),
+ f"The split changed from control 50% / test 50% to control 80% / test 20% on {FIRST_CHANGE}. "
+ "The change came 3 hours after launch, when no group had been exposed.",
+ ),
+ (
+ "variant_zeroed_before_4_groups",
+ _context(THREE_WAY, BLUE_ZEROED, exposed_units=4, counts_groups=True),
+ ("variant_zeroed", ExperimentHealthFindingSeverity.INFO, "E3"),
+ f'Variant "blue" was set to 0% on {FIRST_CHANGE} (from control 34% / test 33% / blue 33% to '
+ "control 50% / test 50% / blue 0%). The change came 3 hours after launch, when 4 groups had been "
+ f"exposed. Move the start date to {FIRST_CHANGE} so the results count only exposures after it. "
+ 'No new users get "blue".',
+ ),
+ (
+ "variant_rolled_out_before_1_person",
+ _context(EVEN, SHIPPED, groups=TARGETED, exposed_units=1),
+ ("variant_rolled_out", ExperimentHealthFindingSeverity.WARNING, "E3"),
+ f'Variant "test" was set to 100% of the split on {FIRST_CHANGE}, while the experiment is still '
+ 'running. From then on, the split assigns everyone to variant "test". End the experiment with a '
+ f"conclusion, or restore the split. {MIXED_ASSIGNMENTS}",
+ ),
(
"variant_removed",
_context(THREE_WAY, EVEN),
@@ -185,6 +230,14 @@ class TestSplitChangedMidRun(TestCase):
f"test 50% on {SECOND_CHANGE}. People exposed in between may have switched variants. Move the start "
f"date to {SECOND_CHANGE}, or reset the experiment.",
),
+ (
+ "reverted_before_4_people",
+ _context(EVEN, SHIPPED, EVEN, exposed_between=True, exposed_units=4),
+ ("reverted", ExperimentHealthFindingSeverity.INFO, "E3"),
+ f"The split was set to control 0% / test 100% on {FIRST_CHANGE} and set back to control 50% / "
+ f"test 50% on {SECOND_CHANGE}. The last change came 3 hours after launch, when 4 people had been "
+ f"exposed. Move the start date to {SECOND_CHANGE} so the results count only exposures after it.",
+ ),
(
"reverted_with_unknown_exposures_in_between",
_context(EVEN, SHIPPED, EVEN, exposed_between=None),
@@ -238,6 +291,21 @@ class TestSplitChangedMidRun(TestCase):
"to": "control 80% / test 20%",
"exposed_before_change": None,
"seconds_after_launch": 11040,
+ "units_exposed_before_last_change": None,
+ },
+ ),
+ (
+ "few_exposed",
+ _context(EVEN, UNEVEN, exposed_units=4),
+ {
+ "change_count": 1,
+ "first_changed_at": "2026-09-12T13:04:00+00:00",
+ "last_changed_at": "2026-09-12T13:04:00+00:00",
+ "from": "control 50% / test 50%",
+ "to": "control 80% / test 20%",
+ "exposed_before_change": True,
+ "seconds_after_launch": 11040,
+ "units_exposed_before_last_change": 4,
},
),
(