:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): report flag bucketing changes after launch (#119019)
Marcel Poelker5447be4f7edf
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex 4efc4fb8ae1b..cd027392c99e 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -10571,9 +10571,9 @@ snapshots:scenes-app-experiments--experiment-frequentist-two-variants--light:hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmIIscenes-app-experiments--experiment-health-debug-tab--dark:- hash: v1.k794b7964.da80c2341c1d9168fa6162308de4aff94d1f925de89e45617d40d984626fff77.kyN0nKYMyjL1PI6j0UGQnzhfI_sW0no-H2DnrHBcW10+ hash: v1.k794b7964.d5684a998e1dc7b37b7fe5db45432053b75fae05ca0538c87e0de6ce3c9dc8ca.2Qe4patz9tyjmFAEZsVCkjjqcrNyTbOAZ3aXHr2mg7gscenes-app-experiments--experiment-health-debug-tab--light:- hash: v1.k794b7964.564e4d31874244bc07e7c7cbcb22c3f63cce8d67e3c0d3e719b2e6f46b0d6c43.Px3OMagE9Nd2lyonNDffZ1wuQg94IwFpXFNd0hYR_YE+ hash: v1.k794b7964.3f5dc6355429b4f0fb060a12d6cdbcee422a7a5d31b95fe77450bebe223b5bf8.F_WmHTt3NUbhkJF8UDfZZ_uET6EuHmF6mYZtGXMHBV4scenes-app-experiments--experiment-recordings-dropped-metric--dark:hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiwscenes-app-experiments--experiment-recordings-dropped-metric--light:diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsxindex e4517869939a..85fbe4246d45 100644--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx@@ -65,14 +65,18 @@ const HEALTH_INPUTS: ExperimentHealthInputsApi = {],variants: SPLIT_AFTER,},- split_history: {+ flag_history: {checked: true,not_checked_reason: null,launched_at: '2025-01-06T10:00:00Z',team_timezone: 'UTC',- changes: [{ changed_at: '2025-01-20T14:30:00Z', before: SPLIT_BEFORE, after: SPLIT_AFTER, counted: true }],+ split_changes: [+ { changed_at: '2025-01-20T14:30:00Z', before: SPLIT_BEFORE, after: SPLIT_AFTER, counted: true },+ ],+ bucketing_changes: [],exposed_before_first: true,- exposed_between: null,+ split_exposed_between: null,+ bucketing_exposed_between: null,},condition_set_traffic_status: 'read',condition_set_traffic: {diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 349e3e263b23..457ab7017b85 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -9926,6 +9926,7 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch* * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run*/export type ExperimentHealthFindingCodeEnum =| "flag_off_while_running"@@ -9938,7 +9939,8 @@ export namespace Schemas {| "zero_exposures"| "forced_variant_release_condition"| "flag_variants_mismatch"- | "split_changed_mid_run";+ | "split_changed_mid_run"+ | "bucketing_changed_mid_run";/*** * `critical` - Critical severity* * `warning` - Warning severity@@ -9982,6 +9984,7 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch* * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run*/code: ExperimentHealthFindingCodeEnum;/**diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.pyindex 49ec281a8959..af4067b6c6b3 100644--- a/products/experiments/backend/facade/api.py+++ b/products/experiments/backend/facade/api.py@@ -131,7 +131,7 @@ def get_experiment_health_findings(*, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess) -> list[ExperimentHealthFinding]:"""The findings of the health checks that read the experiment, its flag, its linked metrics, and the- changes of the flag's split since launch and the flag's recent calls, as far as `reader_access` allows."""+ changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""context = load_experiment_health_context(team_id=team_id, experiment_id=experiment_id, reader_access=reader_access)return evaluate_health(context, EXPERIMENT_HEALTH_CHECKS)diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex ba2f1b0ccf9b..cafd2297156e 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -109,6 +109,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"FLAG_VARIANTS_MISMATCH = "flag_variants_mismatch"SPLIT_CHANGED_MID_RUN = "split_changed_mid_run"+ BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"class HealthExposureInputsStatus(LabeledStrEnum):diff --git a/products/experiments/backend/health/bucketing_changes.py b/products/experiments/backend/health/bucketing_changes.pynew file mode 100644index 000000000000..ab9a32d48334--- /dev/null+++ b/products/experiments/backend/health/bucketing_changes.py@@ -0,0 +1,209 @@+"""+Changes of the flag settings besides the split that decide a person's variant draw, read from the flag's activity+log. Pure functions, no I/O.++The flag service hashes the flag key with an identifier: the group key for a flag that aggregates by a group type,+otherwise the device ID or the distinct ID, as `bucketing_identifier` says, and with experience continuity the+person's stored anonymous ID (`hashed_identifier` in rust/feature-flags/src/flags/flag_matching.rs). A change of any+of them gives the people it touches a new draw, so they can get another variant.+"""++from collections.abc import Mapping, Sequence+from datetime import datetime+from enum import StrEnum+from itertools import groupby++from posthog.dataclasses import frozen++from products.experiments.backend.health.context import BucketingChange, BucketingSetting+from products.feature_flags.backend.facade.filters import condition_aggregation+++class BucketingChangeShape(StrEnum):+ # pinned: the finding's subcodes. The page sends them as `finding_variant` on the health finding events.+ KEY_CHANGED = "key_changed"+ AGGREGATION_CHANGED = "aggregation_changed"+ BUCKETING_IDENTIFIER_CHANGED = "bucketing_identifier_changed"+ EXPERIENCE_CONTINUITY_CHANGED = "experience_continuity_changed"+ REVERTED = "reverted"+++_SHAPES = {+ BucketingSetting.KEY: BucketingChangeShape.KEY_CHANGED,+ BucketingSetting.AGGREGATION: BucketingChangeShape.AGGREGATION_CHANGED,+ BucketingSetting.BUCKETING_IDENTIFIER: BucketingChangeShape.BUCKETING_IDENTIFIER_CHANGED,+ BucketingSetting.EXPERIENCE_CONTINUITY: BucketingChangeShape.EXPERIENCE_CONTINUITY_CHANGED,+}+++@frozen+class BucketingNetEffect:+ shape: BucketingChangeShape+ # One change per setting whose value after its last change differs from before its first, at the time of its+ # last change, in the order of BucketingSetting. Empty when every setting was set back.+ net_changes: tuple[BucketingChange, ...]+ # Every setting that changed, also when it was set back, in the order of BucketingSetting.+ changed_settings: tuple[BucketingSetting, ...]+ first: BucketingChange+ last: BucketingChange+ # The saves that changed a setting. One save can change several.+ change_count: int+++def bucketing_changes_from_activity(+ *,+ changed_at: datetime,+ feature_flag_id: int,+ before: Mapping[BucketingSetting, object],+ after: Mapping[BucketingSetting, object],+) -> list[BucketingChange]:+ """The bucketing changes in one activity log row of the flag. A setting the row does not change is absent."""+ changes = []+ for setting in BucketingSetting:+ value_before = _normalized(setting, before.get(setting), feature_flag_id)+ value_after = _normalized(setting, after.get(setting), feature_flag_id)+ if value_before != value_after:+ changes.append(+ BucketingChange(changed_at=changed_at, setting=setting, before=value_before, after=value_after)+ )+ return changes+++def conditions_follow_flag_aggregation(aggregation: object, conditions: Sequence[object]) -> bool:+ """+ Whether no release condition aggregates by person while the flag-level aggregation names a group type.+ `changes_that_apply` relies on this, because it reads only the flag-level aggregation of each save. The flag+ serializer keeps it true: it sets the flag-level aggregation to null when the conditions mix aggregations.+ """+ flag_aggregation = _group_type_index(aggregation)+ return flag_aggregation is None or all(+ condition_aggregation(condition, flag_aggregation) is not None+ for condition in conditions+ if isinstance(condition, dict)+ )+++def changes_that_apply(+ changes: Sequence[BucketingChange],+ *,+ feature_flag_id: int,+ aggregation: object,+ bucketing_identifier: object,+ trust_flag_aggregation: bool,+) -> tuple[BucketingChange, ...]:+ """+ The changes as the flag service reads them, save by save. It hashes the group key for a flag that aggregates by+ a group type, so it does not read the bucketing identifier and persistence there. It hashes the device ID before+ it reads persistence, so it does not read persistence under device ID bucketing either.++ A setting that changes while the flag service does not read it moves nobody at that save. The new value takes+ effect at the save after which the flag service reads the setting again, so the change counts there, from the+ value that the flag service read last.++ `aggregation` and `bucketing_identifier` are the flag's current values. When `trust_flag_aggregation` is false,+ a release condition can aggregate by person under any flag-level aggregation, so the flag-level aggregation does+ not stop the flag service from reading the identifier.+ """+ state: dict[BucketingSetting, str | int | bool | None] = {+ BucketingSetting.AGGREGATION: _normalized(BucketingSetting.AGGREGATION, aggregation, feature_flag_id),+ BucketingSetting.BUCKETING_IDENTIFIER: _normalized(+ BucketingSetting.BUCKETING_IDENTIFIER, bucketing_identifier, feature_flag_id+ ),+ }+ # Each setting holds the `before` of its first change until that change, and its current value without one.+ for change in reversed(changes):+ state[change.setting] = change.before++ def reads(setting: BucketingSetting) -> bool:+ by_person = state[BucketingSetting.AGGREGATION] is None or not trust_flag_aggregation+ match setting:+ case BucketingSetting.BUCKETING_IDENTIFIER:+ return by_person+ case BucketingSetting.EXPERIENCE_CONTINUITY:+ return by_person and state[BucketingSetting.BUCKETING_IDENTIFIER] == "distinct_id"+ case _:+ return True++ last_read = {setting: value for setting, value in state.items() if reads(setting)}+ applied = []+ for changed_at, saved in groupby(changes, key=lambda change: change.changed_at):+ for change in saved:+ state[change.setting] = change.after+ for setting in BucketingSetting:+ if setting not in state or not reads(setting):+ continue+ # A setting that the flag service never read before has no earlier value to count from. The change+ # that made the flag service read it, such as an aggregation change, counts instead.+ if setting in last_read and last_read[setting] != state[setting]:+ applied.append(+ BucketingChange(+ changed_at=changed_at, setting=setting, before=last_read[setting], after=state[setting]+ )+ )+ last_read[setting] = state[setting]+ return tuple(applied)+++def bucketing_net_effect(changes: Sequence[BucketingChange]) -> BucketingNetEffect | None:+ if not changes:+ return None+ net_changes = []+ changed_settings = []+ for setting in BucketingSetting:+ of_setting = [change for change in changes if change.setting == setting]+ if not of_setting:+ continue+ changed_settings.append(setting)+ if of_setting[0].before != of_setting[-1].after:+ net_changes.append(+ BucketingChange(+ changed_at=of_setting[-1].changed_at,+ setting=setting,+ before=of_setting[0].before,+ after=of_setting[-1].after,+ )+ )+ return BucketingNetEffect(+ shape=_SHAPES[net_changes[0].setting] if net_changes else BucketingChangeShape.REVERTED,+ net_changes=tuple(net_changes),+ changed_settings=tuple(changed_settings),+ first=changes[0],+ last=changes[-1],+ change_count=len({change.changed_at for change in changes}),+ )+++def describe_bucketing_value(setting: BucketingSetting, value: str | int | bool | None) -> str:+ """The value as the flag page names it."""+ match setting:+ case BucketingSetting.KEY:+ return f'"{value}"' if isinstance(value, str) else "an unreadable key"+ case BucketingSetting.AGGREGATION:+ return "users" if value is None else "groups"+ case BucketingSetting.BUCKETING_IDENTIFIER:+ return "device ID" if value == "device_id" else "user ID"+ case BucketingSetting.EXPERIENCE_CONTINUITY:+ return "on" if value else "off"+++def _normalized(setting: BucketingSetting, value: object, feature_flag_id: int) -> str | int | bool | None:+ # Each setting is read the way the flag service reads it, so a save that writes the default explicitly is+ # no change.+ match setting:+ case BucketingSetting.KEY:+ if not isinstance(value, str):+ return None+ # Deleting a flag that an experiment uses appends this suffix to its key (`FeatureFlag.tombstoned_key`),+ # and restoring it removes the suffix again. Neither is a new key.+ tombstone = f":deleted:{feature_flag_id}"+ return value.removesuffix(tombstone)+ case BucketingSetting.AGGREGATION:+ return _group_type_index(value)+ case BucketingSetting.BUCKETING_IDENTIFIER:+ return "device_id" if value == "device_id" else "distinct_id"+ case BucketingSetting.EXPERIENCE_CONTINUITY:+ return value is True+++def _group_type_index(value: object) -> int | None:+ return value if isinstance(value, int) and not isinstance(value, bool) else Nonediff --git a/products/experiments/backend/health/checks/bucketing_changed.py b/products/experiments/backend/health/checks/bucketing_changed.pynew file mode 100644index 000000000000..6e1605ea36ba--- /dev/null+++ b/products/experiments/backend/health/checks/bucketing_changed.py@@ -0,0 +1,239 @@+"""+A flag setting that decides the variant draw changed after launch while people were already exposed: the key, the+group type the flag matches by, the bucketing identifier, or persistence across authentication. Pure functions, no+I/O: `health/exposure_gate.py` loads the changes and the exposure answers.++People exposed before such a change get a new draw, so they can get another variant after it.+"""++from datetime import UTC++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFinding,+ ExperimentHealthFindingActionKind,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.bucketing_changes import (+ BucketingChangeShape,+ BucketingNetEffect,+ bucketing_net_effect,+ describe_bucketing_value,+)+from products.experiments.backend.health.checks.change_wording import (+ delay,+ duration,+ not_checked_because,+ not_checked_next_step,+ suggested_start,+ the_change,+)+from products.experiments.backend.health.checks.split_changed import NOT_CHECKED+from products.experiments.backend.health.context import (+ BucketingChange,+ BucketingSetting,+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,+ HealthContext,+)+from products.experiments.backend.health.split_changes import format_change_time++_TITLE = "The flag's variant assignment changed after launch"+_KEY_TITLE = "The flag key changed after launch"+_REVERTED_INFO_TITLE = "The flag's variant assignment changed and was set back"+_NOT_CHECKED_TITLE = "Variant assignment changes were not checked"++_SETTING_NAMES = {+ BucketingSetting.KEY: "the flag key",+ BucketingSetting.AGGREGATION: 'the flag\'s "Match by" setting',+ BucketingSetting.BUCKETING_IDENTIFIER: 'the flag\'s "Match by" setting',+ BucketingSetting.EXPERIENCE_CONTINUITY: 'the flag\'s "Persist flag across authentication steps" setting',+}+++def bucketing_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:+ history = ctx.flag_history+ flag = ctx.flag+ if history is None or flag is None or flag.deleted or not ctx.is_running or ctx.archived:+ return None+ # `flag_variants_mismatch` reports a flag without variants, and the variants have to come back first.+ if not any(variant.key for variant in flag.variants):+ return None+ if isinstance(history, FlagHistoryNotChecked):+ return _not_checked(history)+ effect = bucketing_net_effect(history.counted_bucketing_changes)+ if effect is None:+ return None++ if effect.shape == BucketingChangeShape.REVERTED:+ first_at = format_change_time(effect.first.changed_at, history.team_timezone)+ changed = _changed_settings(effect)+ if history.bucketing_exposed_between is False:+ span = duration((effect.last.changed_at - effect.first.changed_at).total_seconds())+ return _finding(+ history,+ effect,+ severity=ExperimentHealthFindingSeverity.INFO,+ title=_REVERTED_INFO_TITLE,+ sentences=[+ f"{changed} changed on {first_at} and {_was(effect)} set back {span} later.",+ "Nobody was exposed in between.",+ ],+ )+ last_at = format_change_time(effect.last.changed_at, history.team_timezone)+ return _finding(+ history,+ effect,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title=_TITLE,+ sentences=[+ f"{changed} changed on {first_at} and {_was(effect)} set back on {last_at}.",+ *_delay(history, effect),+ "People exposed in between got a new variant draw, so they may have switched variants.",+ f"Move the start date to {suggested_start(effect.last.changed_at, history.team_timezone)}, or "+ "reset the experiment.",+ ],+ )++ start_at = suggested_start(effect.last.changed_at, history.team_timezone)+ return _finding(+ history,+ effect,+ severity=ExperimentHealthFindingSeverity.WARNING,+ title=_KEY_TITLE if effect.shape == BucketingChangeShape.KEY_CHANGED else _TITLE,+ sentences=[+ *(_what_changed(net, history.team_timezone) for net in effect.net_changes),+ *_delay(history, effect),+ *_consequence(effect),+ f"Reset the experiment, or move its start date to {start_at}.",+ ],+ )+++def _not_checked(history: FlagHistoryNotChecked) -> ExperimentHealthFinding | None:+ # `split_changed_mid_run` reports a history the reader can't view, for every kind of change.+ if history.reason == FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS:+ return None+ effect = bucketing_net_effect(history.bucketing_changes)+ if effect is None:+ return None+ change = the_change(effect.change_count, "the")+ if effect.shape == BucketingChangeShape.REVERTED:+ at = format_change_time(effect.last.changed_at, history.team_timezone)+ what_changed = [f"{_changed_settings(effect)} changed after launch and {_was(effect)} set back on {at}."]+ else:+ what_changed = [_what_changed(net, history.team_timezone) for net in effect.net_changes]+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.BUCKETING_CHANGED_MID_RUN,+ subcode=NOT_CHECKED,+ severity=ExperimentHealthFindingSeverity.INFO,+ title=_NOT_CHECKED_TITLE,+ detail=" ".join(+ [+ *what_changed,+ f"This check can't tell whether anyone was exposed before {change}, because "+ f"{not_checked_because(history.reason)}.",+ not_checked_next_step(history.reason, change),+ ]+ ),+ evidence={"reason": history.reason.value, **_evidence(history, effect, exposed_before_change=None)},+ actions=(ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),+ diagnostic_ref="A6",+ )+++def _what_changed(change: BucketingChange, team_timezone: str) -> str:+ at = format_change_time(change.changed_at, team_timezone)+ before = describe_bucketing_value(change.setting, change.before)+ after = describe_bucketing_value(change.setting, change.after)+ match change.setting:+ case BucketingSetting.KEY:+ return f"The flag key changed from {before} to {after} on {at}."+ case BucketingSetting.AGGREGATION if change.before is not None and change.after is not None:+ return f"The flag changed to match by another group type on {at}."+ case BucketingSetting.AGGREGATION | BucketingSetting.BUCKETING_IDENTIFIER:+ return f"The flag changed from matching by {before} to matching by {after} on {at}."+ case BucketingSetting.EXPERIENCE_CONTINUITY:+ return f'"Persist flag across authentication steps" was turned {after} on {at}.'+++def _consequence(effect: BucketingNetEffect) -> list[str]:+ match effect.shape:+ case BucketingChangeShape.KEY_CHANGED:+ key = next(change for change in effect.net_changes if change.setting == BucketingSetting.KEY)+ old_key = describe_bucketing_value(BucketingSetting.KEY, key.before)+ return [+ "The key is an input of the variant draw, so every user got a new one.",+ f"Exposures under {old_key} are no longer in the results, and code that still uses {old_key} no "+ "longer gets this flag.",+ ]+ case BucketingChangeShape.AGGREGATION_CHANGED:+ consequence = (+ "Every user and group got a new variant draw, so people exposed before "+ f"{the_change(effect.change_count, 'the')} may have switched variants."+ )+ case BucketingChangeShape.BUCKETING_IDENTIFIER_CHANGED:+ consequence = (+ "Users whose device ID differs from their user ID got a new variant draw, so they may have switched "+ "variants."+ )+ case _:+ consequence = "Users who logged in got a new variant draw, so they may have switched variants."+ return [+ consequence,+ "A sample ratio mismatch or users in more than one variant can come from "+ f"{the_change(effect.change_count, 'this')}.",+ ]+++def _changed_settings(effect: BucketingNetEffect) -> str:+ names = list(dict.fromkeys(_SETTING_NAMES[setting] for setting in effect.changed_settings))+ joined = names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"+ return joined[0].upper() + joined[1:]+++def _was(effect: BucketingNetEffect) -> str:+ return "was" if len({_SETTING_NAMES[setting] for setting in effect.changed_settings}) == 1 else "were"+++def _delay(history: FlagHistory, effect: BucketingNetEffect) -> list[str]:+ return delay(+ launched_at=history.launched_at,+ first_changed_at=effect.first.changed_at,+ exposed_before_first=history.exposed_before_first,+ change_count=effect.change_count,+ )+++def _finding(+ history: FlagHistory,+ effect: BucketingNetEffect,+ *,+ severity: ExperimentHealthFindingSeverity,+ title: str,+ sentences: list[str],+) -> ExperimentHealthFinding:+ return ExperimentHealthFinding(+ code=ExperimentHealthFindingCode.BUCKETING_CHANGED_MID_RUN,+ subcode=effect.shape.value,+ severity=severity,+ title=title,+ detail=" ".join(sentences),+ evidence=_evidence(history, effect, exposed_before_change=history.exposed_before_first),+ actions=(ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),+ diagnostic_ref="A6",+ )+++def _evidence(+ history: FlagHistory | FlagHistoryNotChecked, effect: BucketingNetEffect, *, exposed_before_change: bool | None+) -> dict[str, str | int | bool | None]:+ return {+ "change_count": effect.change_count,+ "first_changed_at": effect.first.changed_at.astimezone(UTC).isoformat(),+ "last_changed_at": effect.last.changed_at.astimezone(UTC).isoformat(),+ "changed_settings": ",".join(effect.changed_settings),+ "exposed_before_change": exposed_before_change,+ "seconds_after_launch": int((effect.first.changed_at - history.launched_at).total_seconds()),+ }diff --git a/products/experiments/backend/health/checks/change_wording.py b/products/experiments/backend/health/checks/change_wording.pynew file mode 100644index 000000000000..b35a2857eab4--- /dev/null+++ b/products/experiments/backend/health/checks/change_wording.py@@ -0,0 +1,71 @@+"""The words that the findings about flag changes after launch share. Pure functions, no I/O."""++from datetime import datetime, timedelta+from typing import Literal++from products.experiments.backend.health.context import FlagHistoryNotCheckedReason+from products.experiments.backend.health.split_changes import format_change_time++SECONDS_PER_DAY = 24 * 60 * 60+++def suggested_start(changed_at: datetime, team_timezone: str) -> str:+ # The start date picker saves whole minutes, so a start inside the minute of the change keeps exposures+ # from before it.+ minute = changed_at.replace(second=0, microsecond=0)+ start = minute if minute == changed_at else minute + timedelta(minutes=1)+ return format_change_time(start, team_timezone)+++def delay(+ *, launched_at: datetime, first_changed_at: datetime, exposed_before_first: bool | None, change_count: int+) -> list[str]:+ # When nobody knows whether anyone was exposed before the change, or the change came days after launch,+ # the time since launch tells the reader how many people the change can have moved.+ seconds = (first_changed_at - launched_at).total_seconds()+ if exposed_before_first is not None and seconds <= SECONDS_PER_DAY:+ return []+ change = "The change" if change_count == 1 else "The first change"+ return [f"{change} came {duration(seconds)} after launch."]+++def the_change(change_count: int, determiner: Literal["the", "this"]) -> str:+ if change_count == 1:+ return f"{determiner} change"+ return "the changes" if determiner == "the" else "these changes"+++def not_checked_because(reason: FlagHistoryNotCheckedReason) -> str:+ match reason:+ case FlagHistoryNotCheckedReason.EXPOSURE_FILTERS:+ return "the exposure criteria filter by properties"+ case FlagHistoryNotCheckedReason.EXPOSURE_ACTION:+ return "exposures come from an action"+ case FlagHistoryNotCheckedReason.EXPOSURE_CRITERIA:+ return "the exposure criteria can't be read"+ case FlagHistoryNotCheckedReason.NO_QUERY_ACCESS:+ return "this API key can't query events"+ case FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS:+ return "you can't view the linked feature flag's history"+++def not_checked_next_step(reason: FlagHistoryNotCheckedReason, the_change: str) -> str:+ if reason == FlagHistoryNotCheckedReason.NO_QUERY_ACCESS:+ return f"Add the query:read scope to this API key, or review {the_change} in the flag history."+ return f"Review {the_change} in the flag history."+++def duration(seconds: float) -> str:+ minutes = int(seconds // 60)+ if minutes < 1:+ return "less than a minute"+ if minutes < 60:+ return _count(minutes, "minute")+ hours = minutes // 60+ if hours < 48:+ return _count(hours, "hour")+ return _count(hours // 24, "day")+++def _count(value: int, unit: str) -> str:+ return f"{value} {unit}" if value == 1 else f"{value} {unit}s"diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.pyindex 5323605d8a87..e9af1652e809 100644--- a/products/experiments/backend/health/checks/flag_state.py+++ b/products/experiments/backend/health/checks/flag_state.py@@ -12,7 +12,7 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,)-from products.experiments.backend.health.context import FlagState, HealthContext, SplitHistoryNotChecked+from products.experiments.backend.health.context import FlagHistoryNotChecked, FlagState, HealthContextfrom products.experiments.backend.health.split_changes import (SplitChangeShape,describe_split,@@ -145,10 +145,12 @@ def _finding(def _rollout_since(ctx: HealthContext) -> str:# The split check leaves this rollout to this finding, so the finding carries the date of the change.- history = ctx.split_history+ history = ctx.flag_historyif history is None:return ""- effect = net_effect(history.changes if isinstance(history, SplitHistoryNotChecked) else history.counted_changes)+ effect = net_effect(+ history.split_changes if isinstance(history, FlagHistoryNotChecked) else history.counted_split_changes+ )if effect is None or effect.shape != SplitChangeShape.VARIANT_ROLLED_OUT:return ""at = format_change_time(effect.last.changed_at, history.team_timezone)diff --git a/products/experiments/backend/health/checks/split_changed.py b/products/experiments/backend/health/checks/split_changed.pyindex d2de7e60e5ad..f81386f3f6b8 100644--- a/products/experiments/backend/health/checks/split_changed.py+++ b/products/experiments/backend/health/checks/split_changed.py@@ -5,7 +5,7 @@ functions, no I/O: `health/exposure_gate.py` loads the changes and the exposurePeople exposed before such a change can get another variant after it, so the results mix two assignments."""-from datetime import UTC, timedelta+from datetime import UTCfrom typing import Literalfrom products.experiments.backend.facade.contracts import (@@ -14,13 +14,21 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,)+from products.experiments.backend.health.checks.change_wording import (+ delay,+ duration,+ not_checked_because,+ not_checked_next_step,+ suggested_start,+ the_change,+)from products.experiments.backend.health.checks.flag_state import is_single_variant_shippedfrom products.experiments.backend.health.context import (+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,FlagState,HealthContext,- SplitHistory,- SplitHistoryNotChecked,- SplitHistoryNotCheckedReason,)from products.experiments.backend.health.split_changes import (SplitChangeShape,@@ -41,25 +49,24 @@ NOT_CHECKED = "not_checked"_TITLE = "The variant split changed after launch"_REVERTED_INFO_TITLE = "The variant split changed and was set back"_NOT_CHECKED_TITLE = "Variant split changes were not checked"-_HISTORY_NOT_CHECKED_TITLE = "The variant split history was not checked"-_SECONDS_PER_DAY = 24 * 60 * 60+_HISTORY_NOT_CHECKED_TITLE = "The flag history was not checked"def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:- history = ctx.split_history+ history = ctx.flag_historyflag = ctx.flagif history is None or flag is None or flag.deleted or not ctx.is_running or ctx.archived:return None# `flag_variants_mismatch` reports a flag without variants, and the variants have to come back first.if not any(variant.key for variant in flag.variants):return None- if isinstance(history, SplitHistoryNotChecked):+ if isinstance(history, FlagHistoryNotChecked):return _not_checked(history, flag)- effect = net_effect(history.counted_changes)+ effect = net_effect(history.counted_split_changes)if effect is None or _reported_as_shipped(effect, flag):return None- if effect.shape == SplitChangeShape.REVERTED and history.exposed_between is False:+ if effect.shape == SplitChangeShape.REVERTED and history.split_exposed_between is False:return _finding(history,effect,@@ -68,7 +75,7 @@ def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:sentences=[*_reverted(history, effect), "Nobody was exposed in between."],)if effect.shape == SplitChangeShape.REVERTED:- start_at = _suggested_start(history, effect)+ start_at = suggested_start(effect.last.changed_at, history.team_timezone)return _finding(history,effect,@@ -90,32 +97,34 @@ def split_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:_what_changed(history, effect),*_delay(history, effect),*_consequence(history, effect),- f"A sample ratio mismatch or users in more than one variant can come from {_the_change(effect, 'this')}.",+ "A sample ratio mismatch or users in more than one variant can come from "+ f"{the_change(effect.change_count, 'this')}.",],)-def _not_checked(history: SplitHistoryNotChecked, flag: FlagState) -> ExperimentHealthFinding | None:- if history.reason == SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS:+def _not_checked(history: FlagHistoryNotChecked, flag: FlagState) -> ExperimentHealthFinding | None:+ # This finding speaks for the bucketing check too, because neither can read the history for this reason.+ if history.reason == FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS:return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SPLIT_CHANGED_MID_RUN,subcode=NOT_CHECKED,severity=ExperimentHealthFindingSeverity.INFO,title=_HISTORY_NOT_CHECKED_TITLE,detail=(- "This check can't see whether the variant split changed after launch, because "- f"{_not_checked_because(history.reason)}. Viewing the history needs viewer access to the flag and, "- "with an API key, the activity_log:read scope."+ "This check can't see whether the flag's variant split, key or bucketing changed after launch, "+ f"because {not_checked_because(history.reason)}. Viewing the history needs viewer access to the "+ "flag and, with an API key, the activity_log:read scope."),evidence={"reason": history.reason.value},# The flag history is the page this caller can't open.actions=(),diagnostic_ref="E3",)- effect = net_effect(history.changes)+ effect = net_effect(history.split_changes)if effect is None or _reported_as_shipped(effect, flag):return None- the_change = _the_change(effect, "the")+ change = the_change(effect.change_count, "the")at = format_change_time(effect.last.changed_at, history.team_timezone)from_split = describe_split(effect.from_split)if effect.shape == SplitChangeShape.REVERTED:@@ -130,11 +139,6 @@ def _not_checked(history: SplitHistoryNotChecked, flag: FlagState) -> Experiment)else:what_changed = f"The variant split changed from {from_split} to {describe_split(effect.to_split)} on {at}."- next_step = (- f"Add the query:read scope to this API key, or review {the_change} in the flag history."- if history.reason == SplitHistoryNotCheckedReason.NO_QUERY_ACCESS- else f"Review {the_change} in the flag history."- )return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SPLIT_CHANGED_MID_RUN,subcode=NOT_CHECKED,@@ -143,9 +147,9 @@ def _not_checked(history: SplitHistoryNotChecked, flag: FlagState) -> Experimentdetail=" ".join([what_changed,- f"This check can't tell whether anyone was exposed before {the_change}, because "- f"{_not_checked_because(history.reason)}.",- next_step,+ f"This check can't tell whether anyone was exposed before {change}, because "+ f"{not_checked_because(history.reason)}.",+ not_checked_next_step(history.reason, change),]),evidence={"reason": history.reason.value, **_change_evidence(history, effect, exposed_before_change=None)},@@ -154,26 +158,12 @@ def _not_checked(history: SplitHistoryNotChecked, flag: FlagState) -> Experiment)-def _not_checked_because(reason: SplitHistoryNotCheckedReason) -> str:- match reason:- case SplitHistoryNotCheckedReason.EXPOSURE_FILTERS:- return "the exposure criteria filter by properties"- case SplitHistoryNotCheckedReason.EXPOSURE_ACTION:- return "exposures come from an action"- case SplitHistoryNotCheckedReason.EXPOSURE_CRITERIA:- return "the exposure criteria can't be read"- case SplitHistoryNotCheckedReason.NO_QUERY_ACCESS:- return "this API key can't query events"- case SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS:- return "you can't view the linked feature flag's history"--def _reported_as_shipped(effect: SplitNetEffect, flag: FlagState) -> bool:# `variant_shipped_while_running` reports this state, and its detail carries the date of the rollout.return effect.shape == SplitChangeShape.VARIANT_ROLLED_OUT and is_single_variant_shipped(flag)-def _what_changed(history: SplitHistory, effect: SplitNetEffect) -> str:+def _what_changed(history: FlagHistory, effect: SplitNetEffect) -> str:from_split = describe_split(effect.from_split)to_split = describe_split(effect.to_split)at = format_change_time(effect.last.changed_at, history.team_timezone)@@ -204,9 +194,9 @@ def _what_changed(history: SplitHistory, effect: SplitNetEffect) -> str:return f"The split changed from {from_split} to {to_split} on {at}."-def _consequence(history: SplitHistory, effect: SplitNetEffect) -> list[str]:- start_at = _suggested_start(history, effect)- the_change = _the_change(effect, "the")+def _consequence(history: FlagHistory, effect: SplitNetEffect) -> list[str]:+ start_at = suggested_start(effect.last.changed_at, history.team_timezone)+ change = the_change(effect.change_count, "the")match effect.shape:case SplitChangeShape.VARIANT_ROLLED_OUT:variant = _rolled_out_variant(effect, starts_sentence=False)@@ -223,7 +213,7 @@ def _consequence(history: SplitHistory, effect: SplitNetEffect) -> list[str]:]case SplitChangeShape.VARIANT_ADDED:return [- f"People exposed before {the_change} may have switched variants.",+ f"People exposed before {change} may have switched variants.",f"Reset the experiment, or move its start date to {start_at}.",]case SplitChangeShape.VARIANT_ZEROED:@@ -231,25 +221,25 @@ def _consequence(history: SplitHistory, effect: SplitNetEffect) -> list[str]:# variant from the analysis therefore leaves mixed assignments in the variants that remain.keys = _quoted(zeroed_variant_keys(effect.from_split, effect.to_split), conjunction="and")return [- f"People exposed before {the_change} may have switched variants, and no new users get {keys}.",+ f"People exposed before {change} may have switched variants, and no new users get {keys}.",f"Reset the experiment, or move its start date to {start_at}.",]case SplitChangeShape.REORDERED:# Restoring the order moves the people exposed since the change a second time.return [- f"The order decides who gets which variant, so people exposed before {the_change} may have "+ f"The order decides who gets which variant, so people exposed before {change} may have ""switched variants.",f"Reset the experiment, or move its start date to {start_at}.",]case _:return [- f"People exposed before {the_change} may have switched variants, so the results mix two assignments.",+ f"People exposed before {change} may have switched variants, so the results mix two assignments.",f"Reset the experiment, or move its start date to {start_at} to analyze only the people exposed "- f"after {the_change}.",+ f"after {change}.",]-def _reverted(history: SplitHistory, effect: SplitNetEffect) -> list[str]:+def _reverted(history: FlagHistory, effect: SplitNetEffect) -> list[str]:first_at = format_change_time(effect.first.changed_at, history.team_timezone)last_at = format_change_time(effect.last.changed_at, history.team_timezone)from_split = describe_split(effect.from_split)@@ -259,24 +249,23 @@ def _reverted(history: SplitHistory, effect: SplitNetEffect) -> list[str]:f"and was set back to {from_split}."]changed_to = describe_split(effect.first.after)- if history.exposed_between is False:- span = _duration((effect.last.changed_at - effect.first.changed_at).total_seconds())+ if history.split_exposed_between is False:+ span = duration((effect.last.changed_at - effect.first.changed_at).total_seconds())return [f"The split was set to {changed_to} on {first_at} and set back {span} later."]return [f"The split was set to {changed_to} on {first_at} and set back to {from_split} on {last_at}."]-def _delay(history: SplitHistory, effect: SplitNetEffect) -> list[str]:- # When nobody knows whether anyone was exposed before the change, or the change came days after launch,- # the time since launch tells the reader how many people the change can have moved.- seconds = _seconds_after_launch(history, effect)- if history.exposed_before_first is not None and seconds <= _SECONDS_PER_DAY:- return []- change = "The change" if effect.change_count == 1 else "The first change"- return [f"{change} came {_duration(seconds)} after launch."]+def _delay(history: FlagHistory, effect: SplitNetEffect) -> list[str]:+ return delay(+ launched_at=history.launched_at,+ first_changed_at=effect.first.changed_at,+ exposed_before_first=history.exposed_before_first,+ change_count=effect.change_count,+ )def _finding(- history: SplitHistory,+ history: FlagHistory,effect: SplitNetEffect,*,severity: ExperimentHealthFindingSeverity,@@ -298,7 +287,7 @@ def _finding(def _change_evidence(- history: SplitHistory | SplitHistoryNotChecked, effect: SplitNetEffect, *, exposed_before_change: bool | None+ history: FlagHistory | FlagHistoryNotChecked, effect: SplitNetEffect, *, exposed_before_change: bool | None) -> dict[str, str | int | bool | None]:return {"change_count": effect.change_count,@@ -307,29 +296,10 @@ def _change_evidence("from": describe_split(effect.from_split),"to": describe_split(effect.to_split),"exposed_before_change": exposed_before_change,- "seconds_after_launch": int(_seconds_after_launch(history, effect)),+ "seconds_after_launch": int((effect.first.changed_at - history.launched_at).total_seconds()),}-def _suggested_start(history: SplitHistory, effect: SplitNetEffect) -> str:- # The start date picker saves whole minutes, so a start inside the minute of the change keeps exposures- # from before it.- changed_at = effect.last.changed_at- minute = changed_at.replace(second=0, microsecond=0)- start = minute if minute == changed_at else minute + timedelta(minutes=1)- return format_change_time(start, history.team_timezone)---def _seconds_after_launch(history: SplitHistory | SplitHistoryNotChecked, effect: SplitNetEffect) -> float:- return (effect.first.changed_at - history.launched_at).total_seconds()---def _the_change(effect: SplitNetEffect, determiner: Literal["the", "this"]) -> str:- if effect.change_count == 1:- return f"{determiner} change"- return "the changes" if determiner == "the" else "these changes"--def _rolled_out_variant(effect: SplitNetEffect, *, starts_sentence: bool) -> str:key = rolled_out_variant_key(effect.to_split)if key:@@ -351,19 +321,3 @@ def _quoted(keys: list[str], *, conjunction: Literal["and", "or"]) -> str:if len(quoted) <= 1:return "".join(quoted)return f"{', '.join(quoted[:-1])} {conjunction} {quoted[-1]}"---def _duration(seconds: float) -> str:- minutes = int(seconds // 60)- if minutes < 1:- return "less than a minute"- if minutes < 60:- return _count(minutes, "minute")- hours = minutes // 60- if hours < 48:- return _count(hours, "hour")- return _count(hours // 24, "day")---def _count(value: int, unit: str) -> str:- return f"{value} {unit}" if value == 1 else f"{value} {unit}s"diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex df12b3dbcacd..592d80c0e02c 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -27,25 +27,57 @@ class SplitChange:after: Split+class BucketingSetting(StrEnum):+ # The flag settings besides the split that are inputs of the variant draw, in the order the findings name them.+ KEY = "key"+ AGGREGATION = "aggregation"+ BUCKETING_IDENTIFIER = "bucketing_identifier"+ EXPERIENCE_CONTINUITY = "experience_continuity"+++@frozen+class BucketingChange:+ changed_at: datetime+ setting: BucketingSetting+ # The key as a string, the aggregation as a group type index or None for persons, the identifier as+ # "distinct_id" or "device_id", the continuity as a bool.+ before: str | int | bool | None+ after: str | int | bool | None++@frozen-class SplitHistory:- """The changes of the flag's split after launch that can have moved exposed people to another variant."""+class FlagHistory:+ """The changes of the flag after launch that can have moved exposed people, and whether anyone was exposed+ before them. A change before the first exposure moved nobody, so it does not count."""launched_at: datetimeteam_timezone: str- # Every split change since launch. The counted changes are its tail.- changes: tuple[SplitChange, ...]- # From the first change with an exposure before it, or with an unknown answer, to the last change.- # Empty when nobody was exposed before any change: those changes moved nobody.- counted_changes: tuple[SplitChange, ...]+ split_changes: tuple[SplitChange, ...]+ bucketing_changes: tuple[BucketingChange, ...]+ # The time of the first change, of any kind, with an exposure before it or an unknown answer. Every change+ # from then on counts. None when nobody was exposed before any change.+ counted_since: datetime | None# None when unknown: the exposure query failed, or the request ran out of queries or time.exposed_before_first: bool | None- # Asked only when the last change sets the split back to what it was before the first. None when not- # asked, or unknown.- exposed_between: bool | None+ # Asked per kind only when its last counted change sets back what its first changed. None when not asked,+ # or unknown.+ split_exposed_between: bool | None+ bucketing_exposed_between: bool | None++ @property+ def counted_split_changes(self) -> tuple[SplitChange, ...]:+ return _counted(self.split_changes, self.counted_since)++ @property+ def counted_bucketing_changes(self) -> tuple[BucketingChange, ...]:+ return _counted(self.bucketing_changes, self.counted_since)+++def _counted[C: (SplitChange, BucketingChange)](changes: tuple[C, ...], since: datetime | None) -> tuple[C, ...]:+ return () if since is None else tuple(change for change in changes if change.changed_at >= since)-class SplitHistoryNotCheckedReason(StrEnum):+class FlagHistoryNotCheckedReason(StrEnum):# pinned: the finding's `evidence.reason`.NO_HISTORY_ACCESS = "no_history_access"NO_QUERY_ACCESS = "no_query_access"@@ -55,14 +87,15 @@ class SplitHistoryNotCheckedReason(StrEnum):@frozen-class SplitHistoryNotChecked:- """A split history that was not compared with the exposures, and the reason."""+class FlagHistoryNotChecked:+ """A flag history that was not compared with the exposures, and the reason. Every change counts."""- reason: SplitHistoryNotCheckedReason+ reason: FlagHistoryNotCheckedReasonlaunched_at: datetimeteam_timezone: str- # Every split change since launch. Empty for NO_HISTORY_ACCESS, which does not read them.- changes: tuple[SplitChange, ...]+ # Every change since launch. Empty for NO_HISTORY_ACCESS, which does not read them.+ split_changes: tuple[SplitChange, ...]+ bucketing_changes: tuple[BucketingChange, ...]@frozen@@ -127,7 +160,7 @@ class HealthContext:secondary_metric_count: intexposures: ExposureTotals | None# None when not loaded. Only the health endpoint loads it, because it can query the exposures.- split_history: SplitHistory | SplitHistoryNotChecked | None+ flag_history: FlagHistory | FlagHistoryNotChecked | None# None when not loaded or unknown. Only the health endpoint loads it, because it queries the events.condition_set_traffic: ConditionSetTraffic | None# The variants that the analysis leaves out of the results.@@ -217,7 +250,7 @@ def load_health_context(experiment: Experiment,exposures: ExposureTotals | None = None,*,- split_history: SplitHistory | SplitHistoryNotChecked | None = None,+ flag_history: FlagHistory | FlagHistoryNotChecked | None = None,) -> HealthContext:shared_metric_roles = [saved_metric_link_role(link) for link in saved_metric_links(experiment)]return HealthContext(@@ -231,7 +264,7 @@ def load_health_context(primary_metric_count=len(experiment.metrics or []) + shared_metric_roles.count("primary"),secondary_metric_count=len(experiment.metrics_secondary or []) + shared_metric_roles.count("secondary"),exposures=exposures,- split_history=split_history,+ flag_history=flag_history,condition_set_traffic=None,excluded_variant_keys=frozenset(experiment.excluded_variants or []),)diff --git a/products/experiments/backend/health/exposure_gate.py b/products/experiments/backend/health/exposure_gate.pyindex 738a88864a0f..f8ee025a0c3a 100644--- a/products/experiments/backend/health/exposure_gate.py+++ b/products/experiments/backend/health/exposure_gate.py@@ -1,6 +1,6 @@"""-Loads the split changes of a running experiment's flag since launch, and asks ClickHouse whether anyone was-exposed before each change. A change before the first exposure moved nobody, so it does not count.+Loads the changes of a running experiment's flag since launch that can move exposed people: the split and the+settings that decide the variant draw. Then asks ClickHouse whether anyone was exposed before each change. A change before the first exposure moved nobody, so it does not count.The answer for a past window does not change, so it is cached per window and shared by every reader. A slowanswer is then paid once per change, not on every load of the experiment page. A failed query is cached as@@ -8,17 +8,19 @@ unknown too, so a scan that is too large for a request runs at most once a day.Only exposure criteria that name an event are asked about. The answer for a property filter or an action candepend on what each reader may read, and can change as the filter matches other events over time, so no cached-answer for them is safe to share. The split changes of those experiments are reported as not checked.+answer for them is safe to share. The changes of those experiments are reported as not checked."""import jsonimport hashlib+from bisect import bisect_leftfrom collections.abc import Sequence+from dataclasses import replacefrom datetime import datetime, timedeltafrom time import monotonicfrom django.core.cache import cache-from django.db.models import F, Func, JSONField, Value+from django.db.models import F, Func, JSONField, Q, Valuefrom django.db.models.fields.json import KTfrom django.utils import timezone@@ -43,11 +45,22 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.models.activity_logging.activity_log import ActivityLogfrom products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess+from products.experiments.backend.health.bucketing_changes import (+ BucketingChangeShape,+ bucketing_changes_from_activity,+ bucketing_net_effect,+ changes_that_apply,+ conditions_follow_flag_aggregation,+)from products.experiments.backend.health.context import (+ BucketingChange,+ BucketingSetting,+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,+ Split,SplitChange,- SplitHistory,- SplitHistoryNotChecked,- SplitHistoryNotCheckedReason,+ parse_variants,)from products.experiments.backend.health.split_changes import (same_assignment,@@ -72,8 +85,13 @@ _ANSWER_CACHE_SECONDS = int(timedelta(days=30).total_seconds())_TOO_EXPENSIVE_CACHE_SECONDS = int(timedelta(days=1).total_seconds())_FAILED_CACHE_SECONDS = int(timedelta(hours=1).total_seconds())UNKNOWN = "unknown"-_BEFORE_VARIANTS_PATH = '$.changes[*] ? (@.field == "filters").before.multivariate.variants'-_AFTER_VARIANTS_PATH = '$.changes[*] ? (@.field == "filters").after.multivariate.variants'+# The activity log fields of a flag save that can move exposed people.+_FILTERS = "filters"+_SETTING_FIELDS = {+ BucketingSetting.KEY: "key",+ BucketingSetting.BUCKETING_IDENTIFIER: "bucketing_identifier",+ BucketingSetting.EXPERIENCE_CONTINUITY: "ensure_experience_continuity",+}# Events can arrive after the window ends, so a "nobody" answer for a recent window can still change._SETTLED_AFTER = timedelta(hours=1)# A recent "nobody" is kept briefly, so page loads in that hour do not each run the scan again.@@ -86,15 +104,15 @@ _TOO_EXPENSIVE_ERRORS = ()-def load_split_history(+def load_flag_history(experiment: Experiment, *, reader_access: ExperimentHealthReaderAccess-) -> SplitHistory | SplitHistoryNotChecked | None:+) -> FlagHistory | FlagHistoryNotChecked | None:"""- The split history of a running, not archived experiment whose flag exists and is not deleted, or None.+ The flag history of a running, not archived experiment whose flag exists and is not deleted, or None.Never raises: the health endpoint serves the other findings when this fails."""try:- return _load_split_history(experiment, reader_access)+ return _load_flag_history(experiment, reader_access)except Exception as error:capture_exception(error)return None@@ -114,14 +132,14 @@ def cache_failed_scan(cache_key: str, error: Exception) -> None:def query_any_exposure(experiment: Experiment,*,+ flag_key: str,window_start: datetime,window_end: datetime,variant_keys: Sequence[str],default_exposure_event: str,) -> bool:- """Whether anyone was exposed to one of the variant keys in `[window_start, window_end)`."""+ """Whether anyone was exposed to one of the variant keys of the flag key in `[window_start, window_end)`."""team = experiment.team- flag_key = experiment.get_feature_flag_key()_, variant_property = get_exposure_event_and_property(flag_key, experiment.exposure_criteria, default_exposure_event=default_exposure_event)@@ -163,6 +181,8 @@ def query_any_exposure(class _Window:start: datetimeend: datetime+ # The flag's key and variants while the window lasted. Exposures carry the key they were served under.+ flag_key: strvariant_keys: tuple[str, ...]@@ -203,6 +223,7 @@ class _ExposureGate:try:answer = query_any_exposure(self._experiment,+ flag_key=window.flag_key,window_start=window.start,window_end=window.end,variant_keys=window.variant_keys,@@ -225,7 +246,7 @@ class _ExposureGate:self._launched_at.isoformat(),window.start.isoformat(),window.end.isoformat(),- experiment.get_feature_flag_key(),+ window.flag_key,sorted(window.variant_keys),experiment.exposure_criteria,self._default_exposure_event,@@ -237,9 +258,13 @@ class _ExposureGate:return f"experiment_split_exposure:{experiment.team_id}:{experiment.id}:{digest}"-def _load_split_history(+class _ConditionAggregationMismatch(Exception):+ """A release condition aggregates by person while the flag-level aggregation names a group type."""+++def _load_flag_history(experiment: Experiment, reader_access: ExperimentHealthReaderAccess-) -> SplitHistory | SplitHistoryNotChecked | None:+) -> FlagHistory | FlagHistoryNotChecked | None:launched_at = experiment.start_date# django-stubs types the id as int, but an unsaved instance can carry None.feature_flag_id: int | None = experiment.feature_flag_id@@ -249,115 +274,249 @@ def _load_split_history(return Noneteam_timezone = experiment.team.timezoneif not reader_access.can_read_flag_history:- return SplitHistoryNotChecked(- reason=SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS,+ return FlagHistoryNotChecked(+ reason=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS,launched_at=launched_at,team_timezone=team_timezone,- changes=(),+ split_changes=(),+ bucketing_changes=(),)- changes = tuple(- _split_changes_since(team_id=experiment.team_id, feature_flag_id=feature_flag_id, since=launched_at)+ changes = _flag_changes_since(team_id=experiment.team_id, feature_flag_id=feature_flag_id, since=launched_at)+ flag = experiment.feature_flag+ trust_flag_aggregation = conditions_follow_flag_aggregation(flag.aggregation_group_type_index, flag.conditions)+ relies_on_flag_aggregation = any(+ change.setting in (BucketingSetting.BUCKETING_IDENTIFIER, BucketingSetting.EXPERIENCE_CONTINUITY)+ for change in changes.bucketing)- if not changes:- return SplitHistory(- launched_at=launched_at,- team_timezone=team_timezone,- changes=(),- counted_changes=(),- exposed_before_first=False,- exposed_between=None,+ if relies_on_flag_aggregation and not trust_flag_aggregation:+ # The check then counts every identifier change, so it can report one that moved nobody.+ capture_exception(+ _ConditionAggregationMismatch(), {"team_id": experiment.team_id, "feature_flag_id": feature_flag_id})+ # A change the flag service does not read moved nobody, so it neither counts nor needs an exposure window.+ changes = replace(+ changes,+ bucketing=changes_that_apply(+ changes.bucketing,+ feature_flag_id=feature_flag_id,+ aggregation=flag.aggregation_group_type_index,+ bucketing_identifier=flag.bucketing_identifier,+ trust_flag_aggregation=trust_flag_aggregation,+ ),+ )+ history = FlagHistory(+ launched_at=launched_at,+ team_timezone=team_timezone,+ split_changes=changes.split,+ bucketing_changes=changes.bucketing,+ counted_since=None,+ exposed_before_first=False,+ split_exposed_between=None,+ bucketing_exposed_between=None,+ )+ if not changes.times:+ return historyreason = _not_checked_reason(experiment, reader_access)if reason is not None:- return SplitHistoryNotChecked(- reason=reason, launched_at=launched_at, team_timezone=team_timezone, changes=changes+ return FlagHistoryNotChecked(+ reason=reason,+ launched_at=launched_at,+ team_timezone=team_timezone,+ split_changes=changes.split,+ bucketing_changes=changes.bucketing,)- starts = (launched_at, *(change.changed_at for change in changes[:-1]))- walk = [- _Window(start=start, end=change.changed_at, variant_keys=split_variant_keys(change.before))- for start, change in zip(starts, changes)- ]+ timeline = _Timeline(+ changes,+ current_key=experiment.get_feature_flag_key(),+ current_split=parse_variants(experiment.feature_flag.variants),+ )+ starts = (launched_at, *changes.times[:-1])+ walk = [timeline.window(start=start, end=end) for start, end in zip(starts, changes.times)]gate = _ExposureGate(experiment, launched_at, prefetch=walk)- counted: tuple[SplitChange, ...] = ()- exposed_before_first: bool | None = False- exposed_between: bool | None = None- for index, window in enumerate(walk):+ for window in walk:exposed_before = gate.exposed(window)if exposed_before is not False:- counted = changes[index:]- exposed_before_first = exposed_before- exposed_between = _exposed_between(gate, counted)+ history = replace(history, counted_since=window.end, exposed_before_first=exposed_before)break-- return SplitHistory(- launched_at=launched_at,- team_timezone=team_timezone,- changes=changes,- counted_changes=counted,- exposed_before_first=exposed_before_first,- exposed_between=exposed_between,+ if history.counted_since is None:+ return history++ split = history.counted_split_changes+ bucketing = bucketing_net_effect(history.counted_bucketing_changes)+ return replace(+ history,+ split_exposed_between=(+ _exposed_between(gate, timeline, start=split[0].changed_at, end=split[-1].changed_at)+ if split and same_assignment(split[0].before, split[-1].after)+ else None+ ),+ bucketing_exposed_between=(+ _exposed_between(gate, timeline, start=bucketing.first.changed_at, end=bucketing.last.changed_at)+ if bucketing and bucketing.shape == BucketingChangeShape.REVERTED+ else None+ ),)def _not_checked_reason(experiment: Experiment, reader_access: ExperimentHealthReaderAccess-) -> SplitHistoryNotCheckedReason | None:+) -> FlagHistoryNotCheckedReason | None:# The criteria come first, because adding the query:read scope would not let the check run for them.try:criteria = normalize_to_exposure_criteria(experiment.exposure_criteria)except pydantic.ValidationError:- return SplitHistoryNotCheckedReason.EXPOSURE_CRITERIA+ return FlagHistoryNotCheckedReason.EXPOSURE_CRITERIAconfig = criteria.exposure_config if criteria else Noneif isinstance(config, ActionsNode):- return SplitHistoryNotCheckedReason.EXPOSURE_ACTION+ return FlagHistoryNotCheckedReason.EXPOSURE_ACTIONif config is not None and config.properties:- return SplitHistoryNotCheckedReason.EXPOSURE_FILTERS+ return FlagHistoryNotCheckedReason.EXPOSURE_FILTERSif not reader_access.can_query_events:- return SplitHistoryNotCheckedReason.NO_QUERY_ACCESS+ return FlagHistoryNotCheckedReason.NO_QUERY_ACCESSreturn None-def _exposed_between(gate: _ExposureGate, counted: Sequence[SplitChange]) -> bool | None:- first = counted[0]- last = counted[-1]- if not same_assignment(first.before, last.after):- return None- in_between = [change.after for change in counted[:-1]]- # A flag without variants serves `true`, so people can be exposed with no variant key to look for.- if any(not split_variant_keys(split) for split in in_between):- return None- variant_keys = tuple(sorted({key for split in in_between for key in split_variant_keys(split)}))- return gate.exposed(_Window(start=first.changed_at, end=last.changed_at, variant_keys=variant_keys))+@frozen+class _FlagChanges:+ split: tuple[SplitChange, ...]+ bucketing: tuple[BucketingChange, ...]++ @property+ def times(self) -> tuple[datetime, ...]:+ # One save can change several things, so it is one point in time.+ return tuple(+ sorted({change.changed_at for change in self.split} | {change.changed_at for change in self.bucketing})+ )-def _split_changes_since(*, team_id: int, feature_flag_id: int, since: datetime) -> list[SplitChange]:+class _Timeline:+ """The flag's key and split at any time since launch, from its changes and its current state."""++ def __init__(self, changes: _FlagChanges, *, current_key: str, current_split: Split) -> None:+ self._changes = changes+ self._key_changes = [change for change in changes.bucketing if change.setting == BucketingSetting.KEY]+ # The changes come in time order, so a binary search finds the first change at or after a time.+ self._key_times = [change.changed_at for change in self._key_changes]+ self._split_times = [change.changed_at for change in changes.split]+ self._current_key = current_key+ self._current_split = current_split++ def window(self, *, start: datetime, end: datetime) -> _Window:+ """The window up to a change, with the key and the split that the flag served before that change."""+ return _Window(+ start=start,+ end=end,+ flag_key=self.key_before(end),+ variant_keys=split_variant_keys(self.split_before(end)),+ )++ def key_before(self, at: datetime) -> str:+ index = bisect_left(self._key_times, at)+ key = self._key_changes[index].before if index < len(self._key_changes) else self._current_key+ return key if isinstance(key, str) else self._current_key++ def split_before(self, at: datetime) -> Split:+ index = bisect_left(self._split_times, at)+ return self._changes.split[index].before if index < len(self._changes.split) else self._current_split++ def points_within(self, *, start: datetime, end: datetime) -> list[datetime]:+ return [at for at in self._changes.times if start < at <= end]+++def _exposed_between(gate: _ExposureGate, timeline: _Timeline, *, start: datetime, end: datetime) -> bool | None:+ """Whether anyone was exposed in `[start, end)`, under any key and split that the flag served then."""+ variant_keys_by_flag_key: dict[str, set[str]] = {}+ for at in timeline.points_within(start=start, end=end):+ split = timeline.split_before(at)+ # A flag without variants serves `true`, so people can be exposed with no variant key to look for.+ if not split_variant_keys(split):+ return None+ variant_keys_by_flag_key.setdefault(timeline.key_before(at), set()).update(split_variant_keys(split))+ answers = []+ for flag_key, variant_keys in variant_keys_by_flag_key.items():+ answer = gate.exposed(+ _Window(start=start, end=end, flag_key=flag_key, variant_keys=tuple(sorted(variant_keys)))+ )+ if answer:+ return True+ answers.append(answer)+ return None if None in answers else False+++_PROJECTED = (+ "before_variants",+ "after_variants",+ "before_aggregation",+ "after_aggregation",+ *(f"{side}_{setting}" for side in ("before", "after") for setting in _SETTING_FIELDS),+)+++def _flag_changes_since(*, team_id: int, feature_flag_id: int, since: datetime) -> _FlagChanges:rows = (ActivityLog.objects.filter(+ _changes_any_field(_FILTERS, *_SETTING_FIELDS.values()),team_id=team_id,scope="FeatureFlag",item_id=str(feature_flag_id),created_at__gt=since,- detail__contains={"changes": [{"field": "filters"}]},).order_by("created_at")- # Only the variants decide the split, and the whole `detail` of a flag edit can be large.+ # Only these values decide who gets which variant, and the whole `detail` of a flag edit can be large..annotate(job_type=KT("detail__trigger__job_type"),- before_variants=_variants_of_filters_change(_BEFORE_VARIANTS_PATH),- after_variants=_variants_of_filters_change(_AFTER_VARIANTS_PATH),+ before_variants=_changed(_FILTERS, "before.multivariate.variants"),+ after_variants=_changed(_FILTERS, "after.multivariate.variants"),+ before_aggregation=_changed(_FILTERS, "before.aggregation_group_type_index"),+ after_aggregation=_changed(_FILTERS, "after.aggregation_group_type_index"),+ **{f"before_{setting}": _changed(field, "before") for setting, field in _SETTING_FIELDS.items()},+ **{f"after_{setting}": _changed(field, "after") for setting, field in _SETTING_FIELDS.items()},)- .values_list("created_at", "job_type", "before_variants", "after_variants")+ .values("created_at", "job_type", *_PROJECTED))- changes = [- split_change_from_activity(- changed_at=changed_at, job_type=job_type, before_variants=before_variants, after_variants=after_variants+ split: list[SplitChange] = []+ bucketing: list[BucketingChange] = []+ for row in rows:+ changed_at = row["created_at"]+ split_change = split_change_from_activity(+ changed_at=changed_at,+ job_type=row["job_type"],+ before_variants=row["before_variants"],+ after_variants=row["after_variants"],)- for changed_at, job_type, before_variants, after_variants in rows- ]- return [change for change in changes if change is not None]+ if split_change is not None:+ split.append(split_change)+ bucketing.extend(+ bucketing_changes_from_activity(+ changed_at=changed_at,+ feature_flag_id=feature_flag_id,+ before=_settings(row, "before"),+ after=_settings(row, "after"),+ )+ )+ return _FlagChanges(split=tuple(split), bucketing=tuple(bucketing))++def _settings(row: dict[str, object], side: str) -> dict[BucketingSetting, object]:+ return {+ BucketingSetting.AGGREGATION: row[f"{side}_aggregation"],+ **{setting: row[f"{side}_{setting}"] for setting in _SETTING_FIELDS},+ }-def _variants_of_filters_change(path: str) -> Func:- return Func(F("detail"), Value(path), function="jsonb_path_query_first", output_field=JSONField())++def _changes_any_field(*fields: str) -> Q:+ query = Q()+ for field in fields:+ query |= Q(detail__contains={"changes": [{"field": field}]})+ return query+++def _changed(field: str, path: str) -> Func:+ return Func(+ F("detail"),+ Value(f'$.changes[*] ? (@.field == "{field}").{path}'),+ function="jsonb_path_query_first",+ output_field=JSONField(),+ )diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.pyindex 59b188b0cea7..b939a8d84470 100644--- a/products/experiments/backend/health/inputs_payload.py+++ b/products/experiments/backend/health/inputs_payload.py@@ -1,4 +1,6 @@+import jsonfrom dataclasses import asdict+from datetime import datetimefrom typing import Anyfrom products.experiments.backend.facade.contracts import ExperimentHealthInputsTrafficStatus@@ -6,10 +8,10 @@ from products.experiments.backend.health.checks.forced_variant import needs_condfrom products.experiments.backend.health.condition_set_traffic import denied_scanned_propertiesfrom products.experiments.backend.health.context import (ConditionSetTraffic,+ FlagHistory,+ FlagHistoryNotChecked,FlagState,Split,- SplitHistory,- SplitHistoryNotChecked,)from products.experiments.backend.health.exposure_inputs import ExposureInputsfrom products.experiments.backend.health.loader import HealthInputs@@ -38,30 +40,54 @@ def _health_inputs_flag(flag: FlagState) -> dict[str, Any]:}-def _health_inputs_split_history(history: SplitHistory | SplitHistoryNotChecked) -> dict[str, Any]:- if isinstance(history, SplitHistory):- # The counted changes are the tail of every change, so their count marks where counting starts.- first_counted = len(history.changes) - len(history.counted_changes)- reason, exposed_before_first, exposed_between = None, history.exposed_before_first, history.exposed_between+def _health_inputs_bucketing_value(value: str | int | bool | None) -> str | None:+ if value is None or isinstance(value, str):+ return value+ return json.dumps(value)+++def _health_inputs_flag_history(history: FlagHistory | FlagHistoryNotChecked) -> dict[str, Any]:+ if isinstance(history, FlagHistory):+ counted_since: datetime | None = history.counted_since+ reason = None+ exposed_before_first = history.exposed_before_first+ exposed_between = {+ "split_exposed_between": history.split_exposed_between,+ "bucketing_exposed_between": history.bucketing_exposed_between,+ }else:- first_counted = len(history.changes)- reason, exposed_before_first, exposed_between = history.reason.value, None, None+ counted_since, reason, exposed_before_first = None, history.reason.value, None+ exposed_between = dict.fromkeys(("split_exposed_between", "bucketing_exposed_between"))++ def counted(changed_at: datetime) -> bool:+ return counted_since is not None and changed_at >= counted_since+return {"checked": reason is None,"not_checked_reason": reason,"launched_at": history.launched_at,"team_timezone": history.team_timezone,- "changes": [+ "split_changes": [{"changed_at": change.changed_at,"before": _health_inputs_split(change.before),"after": _health_inputs_split(change.after),- "counted": index >= first_counted,+ "counted": counted(change.changed_at),+ }+ for change in history.split_changes+ ],+ "bucketing_changes": [+ {+ "changed_at": change.changed_at,+ "setting": change.setting.value,+ "before": _health_inputs_bucketing_value(change.before),+ "after": _health_inputs_bucketing_value(change.after),+ "counted": counted(change.changed_at),}- for index, change in enumerate(history.changes)+ for change in history.bucketing_changes],"exposed_before_first": exposed_before_first,- "exposed_between": exposed_between,+ **exposed_between,}@@ -127,7 +153,7 @@ def health_inputs_payload(inputs: HealthInputs) -> dict[str, Any]:"primary_metric_count": ctx.primary_metric_count,"secondary_metric_count": ctx.secondary_metric_count,"flag": _health_inputs_flag(ctx.flag) if ctx.flag else None,- "split_history": _health_inputs_split_history(ctx.split_history) if ctx.split_history else None,+ "flag_history": _health_inputs_flag_history(ctx.flag_history) if ctx.flag_history else None,"condition_set_traffic_status": _health_inputs_traffic_status(inputs),"condition_set_traffic": _health_inputs_traffic(ctx.condition_set_traffic)if ctx.condition_set_trafficdiff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex 76b99fb989d5..21f569bd8abe 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -6,7 +6,7 @@ from posthog.models.user import Userfrom products.experiments.backend.facade.contracts import ExperimentHealthReaderAccessfrom products.experiments.backend.health.condition_set_traffic import load_condition_set_trafficfrom products.experiments.backend.health.context import HealthContext, load_health_context-from products.experiments.backend.health.exposure_gate import load_split_history+from products.experiments.backend.health.exposure_gate import load_flag_historyfrom products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputsfrom products.experiments.backend.models.experiment import Experimentfrom products.experiments.backend.setup_context import ProjectSetupFacts, build_project_setup_facts@@ -29,7 +29,7 @@ def load_experiment_health_context(*, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess) -> HealthContext:"""The context that the checks of the health endpoint read: the experiment, its flag, its linked metrics, the- changes of the flag's split since launch and the flag's recent calls, as far as `reader_access` allows."""+ changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""return _health_context(_load_experiment(team_id, experiment_id), reader_access)@@ -59,7 +59,7 @@ def _load_experiment(team_id: int, experiment_id: int) -> Experiment:def _health_context(experiment: Experiment, reader_access: ExperimentHealthReaderAccess) -> HealthContext:- split_history = load_split_history(experiment, reader_access=reader_access)- context = load_health_context(experiment, split_history=split_history)+ flag_history = load_flag_history(experiment, reader_access=reader_access)+ context = load_health_context(experiment, flag_history=flag_history)traffic = load_condition_set_traffic(experiment, context, reader_access=reader_access)return replace(context, condition_set_traffic=traffic)diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.pyindex 7b7e88b6cd4b..4ad7ac77128c 100644--- a/products/experiments/backend/health/registry.py+++ b/products/experiments/backend/health/registry.py@@ -4,6 +4,7 @@ from posthog.exceptions_capture import capture_exceptionfrom products.experiments.backend.facade.contracts import ExperimentHealthFindingfrom products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded+from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_runfrom products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.flag_variants import flag_variants_mismatchfrom products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition@@ -18,6 +19,7 @@ HealthCheck = Callable[[HealthContext], ExperimentHealthFinding | None]EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (flag_state,split_changed_mid_run,+ bucketing_changed_mid_run,flag_variants_mismatch,forced_variant_release_condition,no_metric,diff --git a/products/experiments/backend/health/test/test_bucketing_changed.py b/products/experiments/backend/health/test/test_bucketing_changed.pynew file mode 100644index 000000000000..4333bc170abb--- /dev/null+++ b/products/experiments/backend/health/test/test_bucketing_changed.py@@ -0,0 +1,210 @@+from datetime import UTC, datetime, timedelta+from typing import Any++from unittest import TestCase++from parameterized import parameterized++from products.experiments.backend.facade.contracts import (+ ExperimentHealthFindingActionKind,+ ExperimentHealthFindingCode,+ ExperimentHealthFindingSeverity,+)+from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run+from products.experiments.backend.health.context import (+ BucketingChange,+ BucketingSetting,+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,+ HealthContext,+ parse_flag_state,+)++LAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)+FIRST_CHANGE_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)+FIRST_CHANGE = "September 12, 2026 at 3:04 PM CEST"+SECOND_CHANGE = "September 12, 2026 at 3:21 PM CEST"+MIXED_ASSIGNMENTS = "A sample ratio mismatch or users in more than one variant can come from this change."+EVEN = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]+KEY = BucketingSetting.KEY+AGGREGATION = BucketingSetting.AGGREGATION+IDENTIFIER = BucketingSetting.BUCKETING_IDENTIFIER+CONTINUITY = BucketingSetting.EXPERIENCE_CONTINUITY++Value = str | int | bool | None+++def _context(+ *changes: tuple[BucketingSetting, Value, Value],+ first_change_at: datetime = FIRST_CHANGE_AT,+ exposed_before_first: bool | None = True,+ exposed_between: bool | None = None,+ is_running: bool = True,+ variants: list[dict[str, Any]] = EVEN,+ not_checked: FlagHistoryNotCheckedReason | None = None,+) -> HealthContext:+ bucketing = tuple(+ BucketingChange(+ changed_at=first_change_at + timedelta(minutes=17 * index), setting=setting, before=before, after=after+ )+ for index, (setting, before, after) in enumerate(changes)+ )+ return HealthContext(+ is_launched=True,+ is_running=is_running,+ has_ended=not is_running,+ is_paused=False,+ archived=False,+ flag=parse_flag_state(+ active=True, deleted=False, groups=[{"properties": [], "rollout_percentage": 100}], variants=variants+ ),+ baseline_variant_key=None,+ primary_metric_count=1,+ secondary_metric_count=0,+ exposures=None,+ flag_history=FlagHistory(+ launched_at=LAUNCHED_AT,+ team_timezone="Europe/Berlin",+ split_changes=(),+ bucketing_changes=bucketing,+ counted_since=bucketing[0].changed_at if bucketing else None,+ exposed_before_first=exposed_before_first,+ split_exposed_between=None,+ bucketing_exposed_between=exposed_between,+ )+ if not_checked is None+ else FlagHistoryNotChecked(+ reason=not_checked,+ launched_at=LAUNCHED_AT,+ team_timezone="Europe/Berlin",+ split_changes=(),+ bucketing_changes=bucketing,+ ),+ condition_set_traffic=None,+ excluded_variant_keys=frozenset(),+ )+++class TestBucketingChangedMidRun(TestCase):+ @parameterized.expand(+ [+ (+ "key_changed",+ _context((KEY, "old-key", "new-key")),+ ("key_changed", ExperimentHealthFindingSeverity.WARNING, "The flag key changed after launch"),+ f'The flag key changed from "old-key" to "new-key" on {FIRST_CHANGE}. The key is an input of the '+ 'variant draw, so every user got a new one. Exposures under "old-key" are no longer in the results, '+ 'and code that still uses "old-key" no longer gets this flag. Reset the experiment, or move its start '+ f"date to {FIRST_CHANGE}.",+ ),+ (+ "bucketing_identifier_changed_days_after_launch",+ _context((IDENTIFIER, "distinct_id", "device_id"), first_change_at=FIRST_CHANGE_AT + timedelta(days=8)),+ (+ "bucketing_identifier_changed",+ ExperimentHealthFindingSeverity.WARNING,+ "The flag's variant assignment changed after launch",+ ),+ "The flag changed from matching by user ID to matching by device ID on September 20, 2026 at 3:04 PM "+ "CEST. The change came 8 days after launch. Users whose device ID differs from their user ID got a "+ f"new variant draw, so they may have switched variants. {MIXED_ASSIGNMENTS} Reset the experiment, or "+ "move its start date to September 20, 2026 at 3:04 PM CEST.",+ ),+ (+ "experience_continuity_changed_with_unknown_exposure",+ _context((CONTINUITY, True, False), exposed_before_first=None),+ (+ "experience_continuity_changed",+ ExperimentHealthFindingSeverity.WARNING,+ "The flag's variant assignment changed after launch",+ ),+ f'"Persist flag across authentication steps" was turned off on {FIRST_CHANGE}. The change came 3 '+ "hours after launch. Users who logged in got a new variant draw, so they may have switched variants. "+ f"{MIXED_ASSIGNMENTS} Reset the experiment, or move its start date to {FIRST_CHANGE}.",+ ),+ (+ "aggregation_changed",+ _context((AGGREGATION, None, 0)),+ (+ "aggregation_changed",+ ExperimentHealthFindingSeverity.WARNING,+ "The flag's variant assignment changed after launch",+ ),+ f"The flag changed from matching by users to matching by groups on {FIRST_CHANGE}. Every user and "+ "group got a new variant draw, so people exposed before the change may have switched variants. "+ f"{MIXED_ASSIGNMENTS} Reset the experiment, or move its start date to {FIRST_CHANGE}.",+ ),+ (+ "two_settings_name_the_key_first",+ _context((CONTINUITY, True, False), (KEY, "old-key", "new-key")),+ ("key_changed", ExperimentHealthFindingSeverity.WARNING, "The flag key changed after launch"),+ f'The flag key changed from "old-key" to "new-key" on {SECOND_CHANGE}. "Persist flag across '+ f'authentication steps" was turned off on {FIRST_CHANGE}. The key is an input of the variant draw, so '+ 'every user got a new one. Exposures under "old-key" are no longer in the results, and code that '+ 'still uses "old-key" no longer gets this flag. Reset the experiment, or move its start date to '+ f"{SECOND_CHANGE}.",+ ),+ (+ "set_back_with_exposures_in_between",+ _context(+ (IDENTIFIER, "distinct_id", "device_id"),+ (IDENTIFIER, "device_id", "distinct_id"),+ exposed_between=True,+ ),+ (+ "reverted",+ ExperimentHealthFindingSeverity.WARNING,+ "The flag's variant assignment changed after launch",+ ),+ f'The flag\'s "Match by" setting changed on {FIRST_CHANGE} and was set back on {SECOND_CHANGE}. '+ "People exposed in between got a new variant draw, so they may have switched variants. Move the "+ f"start date to {SECOND_CHANGE}, or reset the experiment.",+ ),+ (+ "set_back_with_nobody_in_between",+ _context((KEY, "old-key", "new-key"), (KEY, "new-key", "old-key"), exposed_between=False),+ (+ "reverted",+ ExperimentHealthFindingSeverity.INFO,+ "The flag's variant assignment changed and was set back",+ ),+ f"The flag key changed on {FIRST_CHANGE} and was set back 17 minutes later. Nobody was exposed in "+ "between.",+ ),+ (+ "not_checked",+ _context((KEY, "old-key", "new-key"), not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),+ ("not_checked", ExperimentHealthFindingSeverity.INFO, "Variant assignment changes were not checked"),+ f'The flag key changed from "old-key" to "new-key" on {FIRST_CHANGE}. This check can\'t tell whether '+ "anyone was exposed before the change, because the exposure criteria filter by properties. Review "+ "the change in the flag history.",+ ),+ ]+ )+ def test_reports_bucketing_changes(+ self,+ _name: str,+ ctx: HealthContext,+ expected: tuple[str, ExperimentHealthFindingSeverity, str],+ expected_detail: str,+ ) -> None:+ finding = bucketing_changed_mid_run(ctx)++ assert finding is not None+ self.assertEqual(finding.code, ExperimentHealthFindingCode.BUCKETING_CHANGED_MID_RUN)+ self.assertEqual((finding.subcode, finding.severity, finding.title), expected)+ self.assertEqual(finding.detail, expected_detail)+ self.assertEqual(finding.actions, (ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,))++ @parameterized.expand(+ [+ ("no_change", _context()),+ ("ended", _context((KEY, "old-key", "new-key"), is_running=False)),+ ("flag_without_variants", _context((KEY, "old-key", "new-key"), variants=[])),+ # The split check reports a history the reader can't view.+ ("no_history_access", _context(not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS)),+ ]+ )+ def test_reports_nothing(self, _name: str, ctx: HealthContext) -> None:+ self.assertIsNone(bucketing_changed_mid_run(ctx))diff --git a/products/experiments/backend/health/test/test_bucketing_changes.py b/products/experiments/backend/health/test/test_bucketing_changes.pynew file mode 100644index 000000000000..0ebcc85eb2d4--- /dev/null+++ b/products/experiments/backend/health/test/test_bucketing_changes.py@@ -0,0 +1,238 @@+from datetime import UTC, datetime, timedelta++from unittest import TestCase++from parameterized import parameterized++from products.experiments.backend.health.bucketing_changes import (+ BucketingChangeShape,+ bucketing_changes_from_activity,+ bucketing_net_effect,+ changes_that_apply,+ conditions_follow_flag_aggregation,+)+from products.experiments.backend.health.context import BucketingChange, BucketingSetting++CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)+FLAG_ID = 42+KEY = BucketingSetting.KEY+AGGREGATION = BucketingSetting.AGGREGATION+IDENTIFIER = BucketingSetting.BUCKETING_IDENTIFIER+CONTINUITY = BucketingSetting.EXPERIENCE_CONTINUITY+++Value = str | int | bool | None+++def _change(setting: BucketingSetting, before: Value, after: Value, *, minutes: int = 0) -> BucketingChange:+ return BucketingChange(+ changed_at=CHANGED_AT + timedelta(minutes=minutes), setting=setting, before=before, after=after+ )+++class TestBucketingChanges(TestCase):+ @parameterized.expand(+ [+ ("key_renamed", {KEY: "old"}, {KEY: "new"}, [(KEY, "old", "new")]),+ ("key_tombstoned_on_delete", {KEY: "old"}, {KEY: f"old:deleted:{FLAG_ID}"}, []),+ ("key_restored_under_a_new_name", {KEY: f"old:deleted:{FLAG_ID}"}, {KEY: "old-2"}, [(KEY, "old", "old-2")]),+ (+ "identifier_to_device",+ {IDENTIFIER: "distinct_id"},+ {IDENTIFIER: "device_id"},+ [(IDENTIFIER, "distinct_id", "device_id")],+ ),+ ("identifier_default_written", {IDENTIFIER: None}, {IDENTIFIER: "distinct_id"}, []),+ ("continuity_on", {CONTINUITY: False}, {CONTINUITY: True}, [(CONTINUITY, False, True)]),+ ("continuity_default_written", {CONTINUITY: None}, {CONTINUITY: False}, []),+ ("aggregation_to_group", {AGGREGATION: None}, {AGGREGATION: 0}, [(AGGREGATION, None, 0)]),+ ("aggregation_boolean_is_unreadable", {AGGREGATION: None}, {AGGREGATION: False}, []),+ ("row_without_these_settings", {}, {}, []),+ (+ "one_save_changes_two",+ {KEY: "old", CONTINUITY: True},+ {KEY: "new", CONTINUITY: False},+ [(KEY, "old", "new"), (CONTINUITY, True, False)],+ ),+ ]+ )+ def test_bucketing_changes_from_activity(+ self,+ _name: str,+ before: dict[BucketingSetting, object],+ after: dict[BucketingSetting, object],+ expected: list[tuple[BucketingSetting, object, object]],+ ) -> None:+ changes = bucketing_changes_from_activity(+ changed_at=CHANGED_AT, feature_flag_id=FLAG_ID, before=before, after=after+ )++ self.assertEqual([(change.setting, change.before, change.after) for change in changes], expected)++ @parameterized.expand(+ [+ (+ "one_setting",+ [_change(CONTINUITY, False, True)],+ BucketingChangeShape.EXPERIENCE_CONTINUITY_CHANGED,+ [CONTINUITY],+ 1,+ ),+ (+ "key_named_first_whatever_the_order",+ [_change(CONTINUITY, False, True), _change(KEY, "old", "new", minutes=5)],+ BucketingChangeShape.KEY_CHANGED,+ [KEY, CONTINUITY],+ 2,+ ),+ (+ "set_back",+ [+ _change(IDENTIFIER, "distinct_id", "device_id"),+ _change(IDENTIFIER, "device_id", "distinct_id", minutes=6),+ ],+ BucketingChangeShape.REVERTED,+ [],+ 2,+ ),+ (+ "one_set_back_one_kept",+ [+ _change(IDENTIFIER, "distinct_id", "device_id"),+ _change(CONTINUITY, True, False),+ _change(IDENTIFIER, "device_id", "distinct_id", minutes=6),+ ],+ BucketingChangeShape.EXPERIENCE_CONTINUITY_CHANGED,+ [CONTINUITY],+ 2,+ ),+ ]+ )+ def test_bucketing_net_effect(+ self,+ _name: str,+ changes: list[BucketingChange],+ expected_shape: BucketingChangeShape,+ expected_net_settings: list[BucketingSetting],+ expected_change_count: int,+ ) -> None:+ effect = bucketing_net_effect(changes)++ assert effect is not None+ self.assertEqual(effect.shape, expected_shape)+ self.assertEqual([change.setting for change in effect.net_changes], expected_net_settings)+ self.assertEqual(effect.change_count, expected_change_count)++ @parameterized.expand(+ [+ (+ "identifier_on_a_user_flag",+ [_change(IDENTIFIER, "distinct_id", "device_id")],+ None,+ None,+ [_change(IDENTIFIER, "distinct_id", "device_id")],+ ),+ ("identifier_on_a_group_flag", [_change(IDENTIFIER, "distinct_id", "device_id")], 0, None, []),+ (+ "identifier_on_a_group_flag_with_a_person_condition",+ [_change(IDENTIFIER, "distinct_id", "device_id")],+ 0,+ "device_id",+ [_change(IDENTIFIER, "distinct_id", "device_id")],+ False,+ ),+ (+ "continuity_under_user_id",+ [_change(CONTINUITY, False, True)],+ None,+ None,+ [_change(CONTINUITY, False, True)],+ ),+ ("continuity_under_device_id", [_change(CONTINUITY, True, False)], None, "device_id", []),+ ("continuity_on_a_group_flag", [_change(CONTINUITY, False, True)], 0, None, []),+ # The state right after each save decides, not the state today.+ (+ "identifier_before_a_switch_to_groups",+ [_change(IDENTIFIER, "distinct_id", "device_id"), _change(AGGREGATION, None, 0, minutes=5)],+ 0,+ "device_id",+ [_change(IDENTIFIER, "distinct_id", "device_id"), _change(AGGREGATION, None, 0, minutes=5)],+ ),+ (+ "continuity_in_the_save_that_switches_to_device_id",+ [_change(IDENTIFIER, "distinct_id", "device_id"), _change(CONTINUITY, True, False)],+ None,+ "device_id",+ [_change(IDENTIFIER, "distinct_id", "device_id")],+ ),+ (+ "identifier_set_back_while_grouped",+ [+ _change(IDENTIFIER, "distinct_id", "device_id"),+ _change(AGGREGATION, None, 0, minutes=5),+ _change(IDENTIFIER, "device_id", "distinct_id", minutes=10),+ _change(AGGREGATION, 0, None, minutes=15),+ ],+ None,+ "distinct_id",+ [+ _change(IDENTIFIER, "distinct_id", "device_id"),+ _change(AGGREGATION, None, 0, minutes=5),+ _change(AGGREGATION, 0, None, minutes=15),+ _change(IDENTIFIER, "device_id", "distinct_id", minutes=15),+ ],+ ),+ (+ "identifier_changed_while_grouped",+ [+ _change(AGGREGATION, None, 0),+ _change(IDENTIFIER, "distinct_id", "device_id", minutes=5),+ _change(AGGREGATION, 0, None, minutes=10),+ ],+ None,+ "device_id",+ [+ _change(AGGREGATION, None, 0),+ _change(AGGREGATION, 0, None, minutes=10),+ _change(IDENTIFIER, "distinct_id", "device_id", minutes=10),+ ],+ ),+ ("key_on_a_group_flag", [_change(KEY, "old", "new")], 0, None, [_change(KEY, "old", "new")]),+ ]+ )+ def test_changes_that_apply(+ self,+ _name: str,+ changes: list[BucketingChange],+ aggregation: int | None,+ bucketing_identifier: str | None,+ expected: list[BucketingChange],+ trust_flag_aggregation: bool = True,+ ) -> None:+ applied = changes_that_apply(+ changes,+ feature_flag_id=FLAG_ID,+ aggregation=aggregation,+ bucketing_identifier=bucketing_identifier,+ trust_flag_aggregation=trust_flag_aggregation,+ )++ self.assertEqual(list(applied), expected)++ @parameterized.expand(+ [+ ("user_flag_with_a_group_condition", None, [{"aggregation_group_type_index": 0}], True),+ ("group_flag", 0, [{"aggregation_group_type_index": 0}, {"aggregation_group_type_index": 0}], True),+ ("group_flag_with_a_condition_from_before_per_condition_aggregation", 0, [{}], True),+ (+ "group_flag_with_a_person_condition",+ 0,+ [{"aggregation_group_type_index": 0}, {"aggregation_group_type_index": None}],+ False,+ ),+ ]+ )+ def test_conditions_follow_flag_aggregation(+ self, _name: str, aggregation: int | None, conditions: list[dict[str, int | None]], expected: bool+ ) -> None:+ self.assertEqual(conditions_follow_flag_aggregation(aggregation, conditions), expected)diff --git a/products/experiments/backend/health/test/test_exposure_gate.py b/products/experiments/backend/health/test/test_exposure_gate.pyindex da51af703c3a..aba26727fd68 100644--- a/products/experiments/backend/health/test/test_exposure_gate.py+++ b/products/experiments/backend/health/test/test_exposure_gate.py@@ -18,13 +18,14 @@ from posthog.models.user import Userfrom products.experiments.backend.facade.contracts import ExperimentHealthReaderAccessfrom products.experiments.backend.health.context import (- SplitHistory,- SplitHistoryNotChecked,- SplitHistoryNotCheckedReason,+ BucketingSetting,+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,)from products.experiments.backend.health.exposure_gate import (MAX_EXPOSURE_QUERIES,- load_split_history,+ load_flag_history,query_any_exposure,)from products.experiments.backend.hogql_queries.exposure_query_logic import DEFAULT_EXPOSURE_EVENT@@ -67,7 +68,7 @@ def _create_running_experiment()-class TestLoadSplitHistory(BaseTest):+class TestLoadFlagHistory(BaseTest):def setUp(self) -> None:super().setUp()cache.clear()@@ -100,16 +101,31 @@ class TestLoadSplitHistory(BaseTest):)return changed_at+ def _log_change(self, at: datetime, **fields: tuple[Any, Any]) -> None:+ ActivityLog.objects.create(+ team_id=self.team.id,+ scope="FeatureFlag",+ item_id=str(self.experiment.feature_flag_id),+ activity="updated",+ detail={+ "changes": [+ {"type": "FeatureFlag", "action": "changed", "field": field, "before": before, "after": after}+ for field, (before, after) in fields.items()+ ]+ },+ created_at=at,+ )+def _load(self,answers: list[bool | Exception] | Callable[..., bool],*,reader_access: ExperimentHealthReaderAccess = FULL_ACCESS,- ) -> tuple[SplitHistory | SplitHistoryNotChecked | None, MagicMock]:+ ) -> tuple[FlagHistory | FlagHistoryNotChecked | None, MagicMock]:with patch("products.experiments.backend.health.exposure_gate.query_any_exposure", side_effect=answers) as query:- history = load_split_history(self.experiment, reader_access=reader_access)+ history = load_flag_history(self.experiment, reader_access=reader_access)return history, querydef test_changes_before_the_first_exposure_do_not_count(self) -> None:@@ -117,21 +133,74 @@ class TestLoadSplitHistory(BaseTest):history, query = self._load([False, True])- assert isinstance(history, SplitHistory)- self.assertEqual([change.changed_at for change in history.counted_changes], changed_at[1:])+ assert isinstance(history, FlagHistory)+ self.assertEqual([change.changed_at for change in history.counted_split_changes], changed_at[1:])self.assertIs(history.exposed_before_first, True)self.assertEqual([(call.kwargs["window_start"], call.kwargs["window_end"]) for call in query.call_args_list],[(self.launched_at, changed_at[0]), (changed_at[0], changed_at[1])],)+ def test_every_kind_of_change_shares_one_walk(self) -> None:+ first_at = self.launched_at + timedelta(days=1)+ changed_at = [first_at, first_at + timedelta(hours=1), first_at + timedelta(hours=2)]+ self._log_change(changed_at[0], filters=(_filters(_split(50, 50)), _filters(_split(55, 45))))+ self._log_change(changed_at[1], bucketing_identifier=("distinct_id", "device_id"))+ self._log_change(+ changed_at[2],+ filters=(+ {**_filters(_split(55, 45)), "aggregation_group_type_index": None},+ {**_filters(_split(60, 40)), "aggregation_group_type_index": 0},+ ),+ )++ history, query = self._load([False, True])++ assert isinstance(history, FlagHistory)+ self.assertEqual([change.changed_at for change in history.split_changes], [changed_at[0], changed_at[2]])+ self.assertEqual([change.changed_at for change in history.counted_split_changes], changed_at[2:])+ self.assertEqual(+ [(change.changed_at, change.setting) for change in history.counted_bucketing_changes],+ [+ (changed_at[1], BucketingSetting.BUCKETING_IDENTIFIER),+ (changed_at[2], BucketingSetting.AGGREGATION),+ ],+ )+ self.assertEqual(query.call_count, 2)++ def test_each_window_asks_with_the_key_the_flag_had_then(self) -> None:+ first_at = self.launched_at + timedelta(days=1)+ changed_at = [first_at, first_at + timedelta(hours=1)]+ self._log_change(changed_at[0], key=("first-key", "second-key"))+ self._log_change(changed_at[1], key=("second-key", FLAG_KEY))++ history, query = self._load([False, True])++ assert isinstance(history, FlagHistory)+ self.assertEqual([call.kwargs["flag_key"] for call in query.call_args_list], ["first-key", "second-key"])+ self.assertEqual([change.changed_at for change in history.counted_bucketing_changes], changed_at[1:])++ def test_a_key_set_back_asks_between_with_the_key_in_between(self) -> None:+ first_at = self.launched_at + timedelta(days=1)+ changed_at = [first_at, first_at + timedelta(minutes=6)]+ self._log_change(changed_at[0], key=(FLAG_KEY, "renamed"))+ self._log_change(changed_at[1], key=("renamed", FLAG_KEY))++ history, query = self._load([True, False])++ assert isinstance(history, FlagHistory)+ self.assertIs(history.bucketing_exposed_between, False)+ self.assertEqual(+ (query.call_args.kwargs["flag_key"], query.call_args.kwargs["window_start"]), ("renamed", changed_at[0])+ )+def test_no_exposure_before_any_change(self) -> None:self._log_split_changes([_split(50, 50), _split(60, 40), _split(70, 30)])history, _ = self._load([False, False])- assert isinstance(history, SplitHistory)- self.assertEqual(history.counted_changes, ())+ assert isinstance(history, FlagHistory)+ self.assertEqual(history.counted_split_changes, ())@parameterized.expand([@@ -161,24 +230,24 @@ class TestLoadSplitHistory(BaseTest):history, query = self._load([True])- assert isinstance(history, SplitHistory)+ assert isinstance(history, FlagHistory)self.assertIs(history.exposed_before_first, True)self.assertEqual(query.call_count, 1)@parameterized.expand([- ("exposure_filters", _BY_EVENT_PROPERTY, True, True, SplitHistoryNotCheckedReason.EXPOSURE_FILTERS),- ("exposure_action", _BY_ACTION, True, True, SplitHistoryNotCheckedReason.EXPOSURE_ACTION),- ("exposure_criteria", _UNREADABLE, True, True, SplitHistoryNotCheckedReason.EXPOSURE_CRITERIA),- ("no_query_access", {}, True, False, SplitHistoryNotCheckedReason.NO_QUERY_ACCESS),+ ("exposure_filters", _BY_EVENT_PROPERTY, True, True, FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),+ ("exposure_action", _BY_ACTION, True, True, FlagHistoryNotCheckedReason.EXPOSURE_ACTION),+ ("exposure_criteria", _UNREADABLE, True, True, FlagHistoryNotCheckedReason.EXPOSURE_CRITERIA),+ ("no_query_access", {}, True, False, FlagHistoryNotCheckedReason.NO_QUERY_ACCESS),("filters_without_query_access",_BY_EVENT_PROPERTY,True,False,- SplitHistoryNotCheckedReason.EXPOSURE_FILTERS,+ FlagHistoryNotCheckedReason.EXPOSURE_FILTERS,),- ("no_history_access", {}, False, True, SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS),+ ("no_history_access", {}, False, True, FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),])def test_split_changes_that_cannot_be_checked(@@ -187,7 +256,7 @@ class TestLoadSplitHistory(BaseTest):exposure_criteria: dict[str, Any],can_read_flag_history: bool,can_query_events: bool,- expected_reason: SplitHistoryNotCheckedReason,+ expected_reason: FlagHistoryNotCheckedReason,) -> None:self.experiment.exposure_criteria = exposure_criteriaself.experiment.save()@@ -202,10 +271,10 @@ class TestLoadSplitHistory(BaseTest):),)- assert isinstance(history, SplitHistoryNotChecked)+ assert isinstance(history, FlagHistoryNotChecked)self.assertEqual(history.reason, expected_reason)expected_changes = changed_at if can_read_flag_history else []- self.assertEqual([change.changed_at for change in history.changes], expected_changes)+ self.assertEqual([change.changed_at for change in history.split_changes], expected_changes)self.assertEqual(query.call_count, 0)@parameterized.expand(@@ -228,8 +297,8 @@ class TestLoadSplitHistory(BaseTest):history, _ = self._load([], reader_access=reader_access)- assert isinstance(history, SplitHistory)- self.assertEqual(history.counted_changes, ())+ assert isinstance(history, FlagHistory)+ self.assertEqual(history.counted_split_changes, ())@parameterized.expand([@@ -244,11 +313,12 @@ class TestLoadSplitHistory(BaseTest):history, query = self._load([True, False])- assert isinstance(history, SplitHistory)- self.assertIs(history.exposed_between, False)+ assert isinstance(history, FlagHistory)+ self.assertIs(history.split_exposed_between, False)self.assertEqual(query.call_args.kwargs,{+ "flag_key": FLAG_KEY,"window_start": changed_at[0],"window_end": changed_at[1],"variant_keys": ("control", "test"),@@ -272,8 +342,8 @@ class TestLoadSplitHistory(BaseTest):history, _ = self._load([False, error])next_load, next_query = self._load([True])- assert isinstance(history, SplitHistory) and isinstance(next_load, SplitHistory)- self.assertEqual([change.changed_at for change in history.counted_changes], changed_at[1:])+ assert isinstance(history, FlagHistory) and isinstance(next_load, FlagHistory)+ self.assertEqual([change.changed_at for change in history.counted_split_changes], changed_at[1:])self.assertIsNone(history.exposed_before_first)self.assertIsNone(next_load.exposed_before_first)self.assertEqual(next_query.call_count, 0)@@ -301,9 +371,9 @@ class TestLoadSplitHistory(BaseTest):with patch("products.experiments.backend.health.exposure_gate.monotonic", side_effect=lambda: clock[0]):history, query = self._load(nobody_exposed)- assert isinstance(history, SplitHistory)+ assert isinstance(history, FlagHistory)self.assertEqual(query.call_count, expected_queries)- self.assertEqual(history.counted_changes[0].changed_at, changed_at[expected_queries])+ self.assertEqual(history.counted_split_changes[0].changed_at, changed_at[expected_queries])self.assertIsNone(history.exposed_before_first)def test_cached_walk_reads_the_cache_once(self) -> None:@@ -313,8 +383,8 @@ class TestLoadSplitHistory(BaseTest):with patch("products.experiments.backend.health.exposure_gate.cache", wraps=cache) as cache_spy:history, query = self._load([])- assert isinstance(history, SplitHistory)- self.assertEqual(history.counted_changes, ())+ assert isinstance(history, FlagHistory)+ self.assertEqual(history.counted_split_changes, ())self.assertEqual(query.call_count, 0)self.assertEqual((cache_spy.get_many.call_count, cache_spy.get.call_count), (1, 0))@@ -347,8 +417,8 @@ class TestLoadSplitHistory(BaseTest):history, query = self._load([True])- assert isinstance(history, SplitHistory)- self.assertIsNone(history.exposed_between)+ assert isinstance(history, FlagHistory)+ self.assertIsNone(history.split_exposed_between)self.assertEqual(query.call_count, 1)@@ -403,6 +473,7 @@ class TestQueryAnyExposure(ClickhouseTestMixin, BaseTest):exposed = query_any_exposure(experiment,+ flag_key=FLAG_KEY,window_start=now - timedelta(days=5),window_end=now - timedelta(days=1),variant_keys=["control", "test"],diff --git a/products/experiments/backend/health/test/test_flag_state.py b/products/experiments/backend/health/test/test_flag_state.pyindex 453bf7b121bd..f2426ebf7a51 100644--- a/products/experiments/backend/health/test/test_flag_state.py+++ b/products/experiments/backend/health/test/test_flag_state.py@@ -62,7 +62,7 @@ def _context(primary_metric_count=1,secondary_metric_count=0,exposures=None,- split_history=None,+ flag_history=None,condition_set_traffic=None,excluded_variant_keys=frozenset(),)diff --git a/products/experiments/backend/health/test/test_flag_variants.py b/products/experiments/backend/health/test/test_flag_variants.pyindex 2bd3c4857551..ec1600c7f9b9 100644--- a/products/experiments/backend/health/test/test_flag_variants.py+++ b/products/experiments/backend/health/test/test_flag_variants.py@@ -38,7 +38,7 @@ def _context(primary_metric_count=1,secondary_metric_count=0,exposures=None,- split_history=None,+ flag_history=None,condition_set_traffic=None,excluded_variant_keys=frozenset(),)diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.pyindex 9448e4c16383..6790eb6c75da 100644--- a/products/experiments/backend/health/test/test_forced_variant.py+++ b/products/experiments/backend/health/test/test_forced_variant.py@@ -49,7 +49,7 @@ def _context(primary_metric_count=1,secondary_metric_count=0,exposures=None,- split_history=None,+ flag_history=None,condition_set_traffic=traffic,excluded_variant_keys=excluded,)diff --git a/products/experiments/backend/health/test/test_no_metric.py b/products/experiments/backend/health/test/test_no_metric.pyindex 91e7b7849955..09a2f3b06682 100644--- a/products/experiments/backend/health/test/test_no_metric.py+++ b/products/experiments/backend/health/test/test_no_metric.py@@ -21,7 +21,7 @@ def _context(**changes: Any) -> HealthContext:"primary_metric_count": 0,"secondary_metric_count": 0,"exposures": None,- "split_history": None,+ "flag_history": None,"condition_set_traffic": None,"excluded_variant_keys": frozenset(),}diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.pyindex 6368a640e320..02ae4156c059 100644--- a/products/experiments/backend/health/test/test_registry.py+++ b/products/experiments/backend/health/test/test_registry.py@@ -41,7 +41,7 @@ RUNNING_UNEVEN_SPLIT = HealthContext(primary_metric_count=1,secondary_metric_count=0,exposures=UNEVEN_SPLIT_TOTALS,- split_history=None,+ flag_history=None,condition_set_traffic=None,excluded_variant_keys=frozenset(),)diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex 4a7567e7c62a..4fdc06fa4ad7 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -13,11 +13,11 @@ from products.experiments.backend.facade.contracts import (from products.experiments.backend.health.checks.flag_state import flag_statefrom products.experiments.backend.health.checks.split_changed import split_changed_mid_runfrom products.experiments.backend.health.context import (+ FlagHistory,+ FlagHistoryNotChecked,+ FlagHistoryNotCheckedReason,HealthContext,SplitChange,- SplitHistory,- SplitHistoryNotChecked,- SplitHistoryNotCheckedReason,parse_flag_state,parse_variants,)@@ -57,7 +57,7 @@ def _context(first_change_at: datetime = FIRST_CHANGE_AT,archived: bool = False,deleted: bool = False,- not_checked: SplitHistoryNotCheckedReason | None = None,+ not_checked: FlagHistoryNotCheckedReason | None = None,) -> HealthContext:changes = tuple(SplitChange(@@ -76,17 +76,23 @@ def _context(primary_metric_count=1,secondary_metric_count=0,exposures=None,- split_history=SplitHistory(+ flag_history=FlagHistory(launched_at=launched_at,team_timezone="Europe/Berlin",- changes=changes,- counted_changes=changes,+ split_changes=changes,+ bucketing_changes=(),+ counted_since=changes[0].changed_at if changes else None,exposed_before_first=exposed_before_first,- exposed_between=exposed_between,+ split_exposed_between=exposed_between,+ bucketing_exposed_between=None,)if not_checked is None- else SplitHistoryNotChecked(- reason=not_checked, launched_at=launched_at, team_timezone="Europe/Berlin", changes=changes+ else FlagHistoryNotChecked(+ reason=not_checked,+ launched_at=launched_at,+ team_timezone="Europe/Berlin",+ split_changes=changes,+ bucketing_changes=(),),condition_set_traffic=None,excluded_variant_keys=frozenset(),@@ -248,7 +254,7 @@ class TestSplitChangedMidRun(TestCase):),("not_checked",- _context(EVEN, UNEVEN, not_checked=SplitHistoryNotCheckedReason.EXPOSURE_FILTERS),+ _context(EVEN, UNEVEN, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),{"reason": "exposure_filters","change_count": 1,@@ -262,7 +268,7 @@ class TestSplitChangedMidRun(TestCase):),("not_checked_without_history_access",- _context(EVEN, not_checked=SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS),+ _context(EVEN, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),{"reason": "no_history_access"},),]@@ -277,7 +283,7 @@ class TestSplitChangedMidRun(TestCase):[("exposure_filters",- _context(EVEN, UNEVEN, not_checked=SplitHistoryNotCheckedReason.EXPOSURE_FILTERS),+ _context(EVEN, UNEVEN, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),f"The variant split changed from control 50% / test 50% to control 80% / test 20% on {FIRST_CHANGE}. ""This check can't tell whether anyone was exposed before the change, because the exposure criteria ""filter by properties. Review the change in the flag history.",@@ -288,7 +294,7 @@ class TestSplitChangedMidRun(TestCase):EVEN,_split(("control", 70), ("test", 30)),UNEVEN,- not_checked=SplitHistoryNotCheckedReason.EXPOSURE_ACTION,+ not_checked=FlagHistoryNotCheckedReason.EXPOSURE_ACTION,),"The variant split changed 2 times after launch, from control 50% / test 50% to control 80% / "f"test 20%, last on {SECOND_CHANGE}. This check can't tell whether anyone was exposed before the "@@ -296,26 +302,26 @@ class TestSplitChangedMidRun(TestCase):),("exposure_criteria_after_a_split_set_back",- _context(EVEN, SHIPPED, EVEN, not_checked=SplitHistoryNotCheckedReason.EXPOSURE_CRITERIA),+ _context(EVEN, SHIPPED, EVEN, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_CRITERIA),"The variant split changed 2 times after launch and was set back to control 50% / test 50% on "f"{SECOND_CHANGE}. This check can't tell whether anyone was exposed before the changes, because the ""exposure criteria can't be read. Review the changes in the flag history.",),("no_query_access",- _context(EVEN, UNEVEN, not_checked=SplitHistoryNotCheckedReason.NO_QUERY_ACCESS),+ _context(EVEN, UNEVEN, not_checked=FlagHistoryNotCheckedReason.NO_QUERY_ACCESS),f"The variant split changed from control 50% / test 50% to control 80% / test 20% on {FIRST_CHANGE}. ""This check can't tell whether anyone was exposed before the change, because this API key can't ""query events. Add the query:read scope to this API key, or review the change in the flag history.",),("no_history_access",- _context(EVEN, not_checked=SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS),- "This check can't see whether the variant split changed after launch, because you can't view the "- "linked feature flag's history. Viewing the history needs viewer access to the flag and, with an API "- "key, the activity_log:read scope.",+ _context(EVEN, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),+ "This check can't see whether the flag's variant split, key or bucketing changed after launch, because "+ "you can't view the linked feature flag's history. Viewing the history needs viewer access to the "+ "flag and, with an API key, the activity_log:read scope.",(),- "The variant split history was not checked",+ "The flag history was not checked",),])@@ -354,7 +360,7 @@ class TestSplitChangedMidRun(TestCase):("flag_without_variants_reported_by_flag_variants_mismatch", _context(EVEN, [])),("not_checked_on_an_ended_experiment",- _context(EVEN, status=ENDED, not_checked=SplitHistoryNotCheckedReason.NO_HISTORY_ACCESS),+ _context(EVEN, status=ENDED, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),),])@@ -372,7 +378,7 @@ class TestSplitChangedMidRun(TestCase):),("rollout_not_checked",- _context(EVEN, SHIPPED, not_checked=SplitHistoryNotCheckedReason.EXPOSURE_FILTERS),+ _context(EVEN, SHIPPED, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),FIRST_CHANGE,"control 50% / test 50%",),diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.pyindex 7b352f490fa5..9e93094883da 100644--- a/products/experiments/backend/presentation/serializers.py+++ b/products/experiments/backend/presentation/serializers.py@@ -3150,44 +3150,78 @@ class ExperimentHealthInputsFlagSerializer(serializers.Serializer):variants = ExperimentHealthInputsVariantSerializer(many=True, help_text="The flag's variants and their split.")+_COUNTED_HELP = (+ "Whether the checks count this change: every change counts from the first change, of any kind, with an "+ "exposure before it, or an unknown answer. False for every change when not checked."+)++class ExperimentHealthInputsSplitChangeSerializer(serializers.Serializer):changed_at = serializers.DateTimeField(help_text="When the flag's split changed.")before = ExperimentHealthInputsVariantSerializer(many=True, help_text="The split before the change.")after = ExperimentHealthInputsVariantSerializer(many=True, help_text="The split after the change.")- counted = serializers.BooleanField(+ counted = serializers.BooleanField(help_text=_COUNTED_HELP)+++class ExperimentHealthInputsBucketingChangeSerializer(serializers.Serializer):+ changed_at = serializers.DateTimeField(help_text="When the setting changed.")+ setting = serializers.CharField(help_text=(- "Whether the split-change check counts this change: it and every later change count from the first "- "with an exposure before it, or an unknown answer. False for every change when not checked."+ "The flag setting that decides the variant draw: 'key', 'aggregation' (the group type the flag matches "+ "by), 'bucketing_identifier' or 'experience_continuity' (persist the flag across authentication steps)."))+ before = serializers.CharField(+ allow_null=True,+ help_text=(+ "The value before the change: the key, the group type index, 'distinct_id' or 'device_id', or 'true' or "+ "'false'. Null for an aggregation by persons, or an unreadable key."+ ),+ )+ after = serializers.CharField(allow_null=True, help_text="The value after the change, in the same form.")+ counted = serializers.BooleanField(help_text=_COUNTED_HELP)-class ExperimentHealthInputsSplitHistorySerializer(serializers.Serializer):+class ExperimentHealthInputsFlagHistorySerializer(serializers.Serializer):checked = serializers.BooleanField(- help_text="Whether the split changes were compared with the exposures. False when not_checked_reason is set."+ help_text="Whether the changes were compared with the exposures. False when not_checked_reason is set.")not_checked_reason = serializers.CharField(allow_null=True,help_text=(- "Why the split changes were not compared with the exposures: 'no_history_access', 'no_query_access', "+ "Why the changes were not compared with the exposures: 'no_history_access', 'no_query_access', ""'exposure_filters', 'exposure_action' or 'exposure_criteria'. Null when they were."),)launched_at = serializers.DateTimeField(help_text="When the experiment launched.")- team_timezone = serializers.CharField(help_text="The project's time zone, which the finding uses for times.")- changes = ExperimentHealthInputsSplitChangeSerializer(+ team_timezone = serializers.CharField(help_text="The project's time zone, which the findings use for times.")+ split_changes = ExperimentHealthInputsSplitChangeSerializer(many=True,help_text="Every split change since launch. Empty for 'no_history_access', which does not read them.",)+ bucketing_changes = ExperimentHealthInputsBucketingChangeSerializer(+ many=True,+ help_text=(+ "Every change since launch of a setting that decides the variant draw, one per setting and save. Empty "+ "for 'no_history_access'."+ ),+ )exposed_before_first = serializers.BooleanField(allow_null=True,help_text="Whether anyone was exposed before the first counted change. Null when unknown or not checked.",)- exposed_between = serializers.BooleanField(+ split_exposed_between = serializers.BooleanField(+ allow_null=True,+ help_text=(+ "Whether anyone was exposed between the first and the last counted split change. Asked only when the "+ "last restores the split from before the first. Null when not asked or unknown."+ ),+ )+ bucketing_exposed_between = serializers.BooleanField(allow_null=True,help_text=(- "Whether anyone was exposed between the first and the last change. Asked only when the last change "- "restores the split from before the first. Null when not asked or unknown."+ "Whether anyone was exposed between the first and the last counted bucketing change. Asked only when "+ "every setting was set back. Null when not asked or unknown."),)@@ -3216,10 +3250,10 @@ class ExperimentHealthInputsConditionSetTrafficSerializer(serializers.Serializerclass ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):can_read_flag_history = serializers.BooleanField(- help_text="Whether the reader may read the flag's history, which the split-change check needs."+ help_text="Whether the reader may read the flag's history, which the flag-change checks need.")can_query_events = serializers.BooleanField(- help_text="Whether the reader may query events, which the split-change and forced-variant checks need."+ help_text="Whether the reader may query events, which the flag-change and forced-variant checks need.")denied_event_properties = serializers.ListField(child=serializers.CharField(),@@ -3306,11 +3340,11 @@ class ExperimentHealthInputsSerializer(serializers.Serializer):flag = ExperimentHealthInputsFlagSerializer(allow_null=True, help_text="The experiment's feature flag. Null when it has none.")- split_history = ExperimentHealthInputsSplitHistorySerializer(+ flag_history = ExperimentHealthInputsFlagHistorySerializer(allow_null=True,help_text=(- "The changes of the flag's split since launch. Null when the experiment is not running, is archived, "- "its flag is missing or deleted, or the read failed."+ "The changes of the flag since launch that can move exposed people. Null when the experiment is not "+ "running, is archived, its flag is missing or deleted, or the read failed."),)condition_set_traffic_status = serializers.ChoiceField(diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.pyindex 635610a9ee91..9377faceaadf 100644--- a/products/experiments/backend/test/test_presentation_api.py+++ b/products/experiments/backend/test/test_presentation_api.py@@ -81,6 +81,14 @@ def _make(cls, **attrs):_HEALTH_PAUSED = ("flag_off_while_running", "running_but_flag_disabled")+_REBALANCED = {+ "filters": {+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "multivariate": {+ "variants": [{"key": "control", "rollout_percentage": 80}, {"key": "test", "rollout_percentage": 20}]+ },+ }+}_HEALTH_TRAFFIC = ConditionSetTraffic(since=datetime(2026, 10, 12, tzinfo=UTC),team_timezone="UTC",@@ -736,9 +744,22 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):{"metrics": [_HEALTH_METRIC]},None,[_HEALTH_PAUSED, ("split_changed_mid_run", "rebalanced")],- timedelta(days=1),+ (timedelta(days=1), _REBALANCED),+ ),+ (+ "split_edited_before_launch",+ {"metrics": [_HEALTH_METRIC]},+ None,+ [_HEALTH_PAUSED],+ (-timedelta(days=1), _REBALANCED),+ ),+ (+ "key_renamed_after_launch",+ {"metrics": [_HEALTH_METRIC]},+ None,+ [_HEALTH_PAUSED, ("bucketing_changed_mid_run", "key_changed")],+ (timedelta(days=1), {"key": "health-findings-renamed"}),),- ("split_edited_before_launch", {"metrics": [_HEALTH_METRIC]}, None, [_HEALTH_PAUSED], -timedelta(days=1)),])def test_health_reports_findings(@@ -747,7 +768,7 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):experiment_fields: dict[str, Any],shared_metric_link: dict[str, Any] | None,expected: list[tuple[str, str | None]],- split_edit_from_launch: timedelta | None = None,+ flag_edit: tuple[timedelta, dict[str, Any]] | None = None,) -> None:start_date = timezone.now() - timedelta(days=3)flag = FeatureFlag.objects.create(@@ -791,8 +812,9 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):format="json",)self.assertEqual(link_response.status_code, status.HTTP_200_OK, link_response.json())- if split_edit_from_launch is not None:- self._rebalance_split_at(flag, start_date + split_edit_from_launch)+ if flag_edit is not None:+ from_launch, fields = flag_edit+ self._edit_flag_at(flag, start_date + from_launch, fields)with patch("products.experiments.backend.health.exposure_gate.query_any_exposure", return_value=True):response = self.client.get(f"/api/projects/{self.team.id}/experiments/{experiment.id}/health/")@@ -852,7 +874,7 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):experiment = Experiment.objects.create(team=self.team, name="health-split-access", feature_flag=flag, start_date=start_date)- self._rebalance_split_at(flag, start_date + timedelta(days=1))+ self._edit_flag_at(flag, start_date + timedelta(days=1), _REBALANCED)headers: dict[str, str] = {}if restrict_flag:self._enable_access_control()@@ -1046,7 +1068,7 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):team=self.team, name="health-inputs-split", feature_flag=flag, start_date=start_date)if rebalance:- self._rebalance_split_at(flag, start_date + timedelta(days=1))+ self._edit_flag_at(flag, start_date + timedelta(days=1), _REBALANCED)headers: dict[str, str] = {}if token_scopes is not None:token = generate_random_token_personal()@@ -1062,9 +1084,13 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):)self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)- history = response.json()["split_history"]+ history = response.json()["flag_history"]self.assertEqual(- (history["checked"], history["not_checked_reason"], [change["counted"] for change in history["changes"]]),+ (+ history["checked"],+ history["not_checked_reason"],+ [change["counted"] for change in history["split_changes"]],+ ),expected,)@@ -1118,17 +1144,10 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):metrics=[_HEALTH_METRIC],)- def _rebalance_split_at(self, flag: FeatureFlag, at: datetime) -> None:+ def _edit_flag_at(self, flag: FeatureFlag, at: datetime, fields: dict[str, Any]) -> None:with time_machine.travel(at), self.captureOnCommitCallbacks(execute=True):- flag.filters = {- **flag.filters,- "multivariate": {- "variants": [- {"key": "control", "rollout_percentage": 80},- {"key": "test", "rollout_percentage": 20},- ]- },- }+ for field, value in fields.items():+ setattr(flag, field, value)flag.save()@parameterized.expand(diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex 792e985d0a84..107037fef9dd 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2274,6 +2274,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch* * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run*/export type ExperimentHealthFindingCodeEnumApi =(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]@@ -2290,6 +2291,7 @@ export const ExperimentHealthFindingCodeEnumApi = {ForcedVariantReleaseCondition: 'forced_variant_release_condition',FlagVariantsMismatch: 'flag_variants_mismatch',SplitChangedMidRun: 'split_changed_mid_run',+ BucketingChangedMidRun: 'bucketing_changed_mid_run',} as const/**@@ -2348,7 +2350,8 @@ export interface ExperimentHealthFindingApi {* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch- * * `split_changed_mid_run` - Split Changed Mid Run */+ * * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run */code: ExperimentHealthFindingCodeEnumApi/*** The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.@@ -3065,34 +3068,60 @@ export interface ExperimentHealthInputsSplitChangeApi {before: ExperimentHealthInputsVariantApi[]/** The split after the change. */after: ExperimentHealthInputsVariantApi[]- /** Whether the split-change check counts this change: it and every later change count from the first with an exposure before it, or an unknown answer. False for every change when not checked. */+ /** Whether the checks count this change: every change counts from the first change, of any kind, with an exposure before it, or an unknown answer. False for every change when not checked. */counted: boolean}-export interface ExperimentHealthInputsSplitHistoryApi {- /** Whether the split changes were compared with the exposures. False when not_checked_reason is set. */+export interface ExperimentHealthInputsBucketingChangeApi {+ /** When the setting changed. */+ changed_at: string+ /** The flag setting that decides the variant draw: 'key', 'aggregation' (the group type the flag matches by), 'bucketing_identifier' or 'experience_continuity' (persist the flag across authentication steps). */+ setting: string+ /**+ * The value before the change: the key, the group type index, 'distinct_id' or 'device_id', or 'true' or 'false'. Null for an aggregation by persons, or an unreadable key.+ * @nullable+ */+ before: string | null+ /**+ * The value after the change, in the same form.+ * @nullable+ */+ after: string | null+ /** Whether the checks count this change: every change counts from the first change, of any kind, with an exposure before it, or an unknown answer. False for every change when not checked. */+ counted: boolean+}++export interface ExperimentHealthInputsFlagHistoryApi {+ /** Whether the changes were compared with the exposures. False when not_checked_reason is set. */checked: boolean/**- * Why the split changes were not compared with the exposures: 'no_history_access', 'no_query_access', 'exposure_filters', 'exposure_action' or 'exposure_criteria'. Null when they were.+ * Why the changes were not compared with the exposures: 'no_history_access', 'no_query_access', 'exposure_filters', 'exposure_action' or 'exposure_criteria'. Null when they were.* @nullable*/not_checked_reason: string | null/** When the experiment launched. */launched_at: string- /** The project's time zone, which the finding uses for times. */+ /** The project's time zone, which the findings use for times. */team_timezone: string/** Every split change since launch. Empty for 'no_history_access', which does not read them. */- changes: ExperimentHealthInputsSplitChangeApi[]+ split_changes: ExperimentHealthInputsSplitChangeApi[]+ /** Every change since launch of a setting that decides the variant draw, one per setting and save. Empty for 'no_history_access'. */+ bucketing_changes: ExperimentHealthInputsBucketingChangeApi[]/*** Whether anyone was exposed before the first counted change. Null when unknown or not checked.* @nullable*/exposed_before_first: boolean | null/**- * Whether anyone was exposed between the first and the last change. Asked only when the last change restores the split from before the first. Null when not asked or unknown.+ * Whether anyone was exposed between the first and the last counted split change. Asked only when the last restores the split from before the first. Null when not asked or unknown.+ * @nullable+ */+ split_exposed_between: boolean | null+ /**+ * Whether anyone was exposed between the first and the last counted bucketing change. Asked only when every setting was set back. Null when not asked or unknown.* @nullable*/- exposed_between: boolean | null+ bucketing_exposed_between: boolean | null}/**@@ -3136,9 +3165,9 @@ export interface ExperimentHealthInputsConditionSetTrafficApi {}export interface ExperimentHealthInputsReaderAccessApi {- /** Whether the reader may read the flag's history, which the split-change check needs. */+ /** Whether the reader may read the flag's history, which the flag-change checks need. */can_read_flag_history: boolean- /** Whether the reader may query events, which the split-change and forced-variant checks need. */+ /** Whether the reader may query events, which the flag-change and forced-variant checks need. */can_query_events: boolean/** Event properties the reader is denied. The forced-variant check skips its scan when one of them is a flag call property it reads. */denied_event_properties: string[]@@ -3412,8 +3441,8 @@ export interface ExperimentHealthInputsApi {secondary_metric_count: number/** The experiment's feature flag. Null when it has none. */flag: ExperimentHealthInputsFlagApi | null- /** The changes of the flag's split since launch. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */- split_history: ExperimentHealthInputsSplitHistoryApi | null+ /** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */+ flag_history: ExperimentHealthInputsFlagHistoryApi | null/** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis.** * `read` - Readdiff --git a/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx b/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsxindex 526eab66f9c5..0e12e502218b 100644--- a/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx+++ b/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx@@ -7,7 +7,7 @@ import { experimentLogic } from 'scenes/experiments/experimentLogic'import { SetupFactsTable } from '../../setupInspector/SetupFactsTable'import { formatCount, formatYesNo } from '../../setupInspector/setupInspectorUtils'import { experimentHealthDebugLogic } from './experimentHealthDebugLogic'-import type { HealthDebugReleaseCondition, HealthDebugSplitChange } from './healthDebugInputs'+import type { HealthDebugFlagChange, HealthDebugReleaseCondition } from './healthDebugInputs'import { HealthDebugSection } from './HealthDebugSection'export function HealthDebugFlagTab(): JSX.Element {@@ -15,15 +15,15 @@ export function HealthDebugFlagTab(): JSX.Element {const {flagFacts,releaseConditions,- splitHistoryFacts,- splitChanges,+ flagHistoryFacts,+ flagChanges,trafficFacts,healthInputs,healthInputsLoading,healthInputsError,} = useValues(experimentHealthDebugLogic({ experimentId }))const trafficRead = !!healthInputs?.condition_set_traffic- const splitHistoryChecked = !!healthInputs?.split_history?.checked+ const flagHistoryChecked = !!healthInputs?.flag_history?.checkedreturn (<div className="flex flex-col gap-4">@@ -84,25 +84,26 @@ export function HealthDebugFlagTab(): JSX.Element {/></HealthDebugSection><HealthDebugSection- title="Split changes since launch"- description="What the split-change check read from the flag's history."+ title="Changes since launch"+ description="What the split and bucketing checks read from the flag's history.">- <SetupFactsTable facts={splitHistoryFacts} />- {splitChanges.length > 0 && (- <LemonTable<HealthDebugSplitChange>+ <SetupFactsTable facts={flagHistoryFacts} />+ {flagChanges.length > 0 && (+ <LemonTable<HealthDebugFlagChange>size="small"- dataSource={splitChanges}+ dataSource={flagChanges}rowKey={(change, index) => `${change.changedAt}-${index}`}columns={[{ title: 'Changed at', dataIndex: 'changedAt' },+ { title: 'Changed', dataIndex: 'changed' },{ title: 'Before', dataIndex: 'before' },{ title: 'After', dataIndex: 'after' },- ...(splitHistoryChecked+ ...(flagHistoryChecked? [{title: 'Counts',key: 'counted',- render: (_: unknown, change: HealthDebugSplitChange) =>+ render: (_: unknown, change: HealthDebugFlagChange) =>formatYesNo(change.counted),},]diff --git a/products/experiments/frontend/health/debug/experimentHealthDebugLogic.ts b/products/experiments/frontend/health/debug/experimentHealthDebugLogic.tsindex 263ae12208df..78b1f1c013e7 100644--- a/products/experiments/frontend/health/debug/experimentHealthDebugLogic.ts+++ b/products/experiments/frontend/health/debug/experimentHealthDebugLogic.ts@@ -16,17 +16,17 @@ import type { ExperimentHealthApi, ExperimentHealthInputsApi } from '../../generimport type { SetupFact } from '../../setupInspector/SetupFactsTable'import { type HealthPanelFinding, hoursSinceStart } from '../healthPanelFindings'import {+ type HealthDebugFlagChange,type HealthDebugReleaseCondition,- type HealthDebugSplitChange,type HealthInputsState,type PageMetricCounts,buildExperimentFacts,buildExposureFacts,+ buildFlagChanges,buildFlagFacts,+ buildFlagHistoryFacts,buildMetricFacts,buildReleaseConditions,- buildSplitChanges,- buildSplitHistoryFacts,buildTrafficFacts,} from './healthDebugInputs'import {@@ -54,7 +54,9 @@ export interface experimentHealthDebugLogicValues {activeDomain: HealthDebugDomainexperimentFacts: SetupFact[]exposureFacts: SetupFact[]+ flagChanges: HealthDebugFlagChange[]flagFacts: SetupFact[]+ flagHistoryFacts: SetupFact[]healthDebugChecks: HealthDebugCheck[]healthDebugRawData: Record<string, unknown>healthDebugSummary: SetupFact[]@@ -65,8 +67,6 @@ export interface experimentHealthDebugLogicValues {metricFacts: SetupFact[]pageMetricCounts: PageMetricCountsreleaseConditions: HealthDebugReleaseCondition[]- splitChanges: HealthDebugSplitChange[]- splitHistoryFacts: SetupFact[]trafficFacts: SetupFact[]}@@ -131,8 +131,8 @@ export interface experimentHealthDebugLogicMeta {experimentFacts: (experiment: Experiment, healthInputsState: HealthInputsState) => SetupFact[]flagFacts: (experiment: Experiment, healthInputsState: HealthInputsState) => SetupFact[]releaseConditions: (healthInputs: ExperimentHealthInputsApi | null) => HealthDebugReleaseCondition[]- splitHistoryFacts: (healthInputsState: HealthInputsState) => SetupFact[]- splitChanges: (healthInputs: ExperimentHealthInputsApi | null) => HealthDebugSplitChange[]+ flagHistoryFacts: (healthInputsState: HealthInputsState) => SetupFact[]+ flagChanges: (healthInputs: ExperimentHealthInputsApi | null) => HealthDebugFlagChange[]trafficFacts: (healthInputsState: HealthInputsState) => SetupFact[]metricFacts: (experiment: Experiment,@@ -306,13 +306,13 @@ export const experimentHealthDebugLogic = kea<experimentHealthDebugLogicType>([(s) => [s.healthInputs],(inputs: ExperimentHealthInputsApi | null): HealthDebugReleaseCondition[] => buildReleaseConditions(inputs),],- splitHistoryFacts: [+ flagHistoryFacts: [(s) => [s.healthInputsState],- (state: HealthInputsState): SetupFact[] => buildSplitHistoryFacts(state),+ (state: HealthInputsState): SetupFact[] => buildFlagHistoryFacts(state),],- splitChanges: [+ flagChanges: [(s) => [s.healthInputs],- (inputs: ExperimentHealthInputsApi | null): HealthDebugSplitChange[] => buildSplitChanges(inputs),+ (inputs: ExperimentHealthInputsApi | null): HealthDebugFlagChange[] => buildFlagChanges(inputs),],trafficFacts: [(s) => [s.healthInputsState],diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.tsindex 6578224f928a..c348e349d16b 100644--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts@@ -5,7 +5,7 @@ import { getExperimentStatus } from '../../experimentStatus'import type {ExperimentHealthInputsApi,ExperimentHealthInputsExposuresApi,- ExperimentHealthInputsSplitHistoryApi,+ ExperimentHealthInputsFlagHistoryApi,ExperimentHealthInputsTrafficStatusEnumApi,ExperimentHealthInputsVariantApi,HealthExposureInputsStatusEnumApi,@@ -29,8 +29,10 @@ export interface HealthDebugReleaseCondition {people: number | null}-export interface HealthDebugSplitChange {+export interface HealthDebugFlagChange {changedAt: string+ /** What changed: the split, or a setting that decides the variant draw. */+ changed: stringbefore: stringafter: stringcounted: boolean@@ -88,7 +90,7 @@ export function buildExperimentFacts({label: 'State the checks read',value: serverValue(state, describeState),- help: 'The flag-state, split-change and forced-variant checks only fire for some of these states.',+ help: 'The flag-state, flag-change and forced-variant checks only fire for some of these states.',},{ label: 'Start date', value: formatTimestamp(experiment.start_date) },{@@ -114,7 +116,7 @@ export function buildExperimentFacts(({ reader_access }) =>`Flag history: ${formatYesNo(reader_access.can_read_flag_history).toLowerCase()}. Event queries: ${formatYesNo(reader_access.can_query_events).toLowerCase()}. Denied event properties: ${reader_access.denied_event_properties.join(', ') || 'none'}.`),- help: 'The split-change and forced-variant checks read only what the person loading the page may read. A customer without this access gets other findings than you.',+ help: 'The flag-change and forced-variant checks read only what the person loading the page may read. A customer without this access gets other findings than you.',},]}@@ -150,16 +152,21 @@ export function buildReleaseConditions(inputs: ExperimentHealthInputsApi | null)}))}-function describeSplitHistoryStatus(inputs: ExperimentHealthInputsApi): string {- const history = inputs.split_history+function allFlagChanges(history: ExperimentHealthInputsFlagHistoryApi): { counted: boolean }[] {+ return [...history.split_changes, ...history.bucketing_changes]+}++function describeFlagHistoryStatus(inputs: ExperimentHealthInputsApi): string {+ const history = inputs.flag_historyif (history) {if (!history.checked) {return `Not compared: ${history.not_checked_reason}`}- const total = history.changes.length- const counted = history.changes.filter((change) => change.counted).length+ const changes = allFlagChanges(history)+ const total = changes.length+ const counted = changes.filter((change) => change.counted).lengthif (total === 0) {- return 'No split changes since launch'+ return 'No changes since launch'}if (total === 1) {return counted@@ -182,14 +189,23 @@ function describeSplitHistoryStatus(inputs: ExperimentHealthInputsApi): string {return 'Not read: the read failed'}-export function buildSplitHistoryFacts(state: HealthInputsState): SetupFact[] {- const history: ExperimentHealthInputsSplitHistoryApi | null = state.inputs?.split_history ?? null- const hasCountedChange = !!history?.changes.some((change) => change.counted)+const SET_BACK_FACTS = [+ { kind: 'split_changes', field: 'split_exposed_between', label: 'Exposed between the first and last split change' },+ {+ kind: 'bucketing_changes',+ field: 'bucketing_exposed_between',+ label: 'Exposed between the first and last bucketing change',+ },+] as const++export function buildFlagHistoryFacts(state: HealthInputsState): SetupFact[] {+ const history = state.inputs?.flag_history ?? null+ const hasCountedChange = !!history && allFlagChanges(history).some((change) => change.counted)return [{label: 'Status',- value: serverValue(state, describeSplitHistoryStatus),- help: 'The split-change check compares the changes with the exposures only for an exposure event without property filters.',+ value: serverValue(state, describeFlagHistoryStatus),+ help: 'The flag-change checks compare the changes with the exposures only for an exposure event without property filters. Every change counts from the first one, of any kind, with an exposure before it.',},...(history? [{ label: 'Launched at', value: `${formatTimestamp(history.launched_at)} (${history.team_timezone})` }]@@ -200,23 +216,49 @@ export function buildSplitHistoryFacts(state: HealthInputsState): SetupFact[] {label: 'Exposed before the first counted change',value: yesNoUnknown(history.exposed_before_first),},- {- label: 'Exposed between the first and last change',- value: yesNoUnknown(history.exposed_between),- help: 'Asked only when the last change sets the split back to what it was before the first.',- },+ ...SET_BACK_FACTS.filter(({ kind }) => history[kind].some((change) => change.counted)).map(+ ({ field, label }) => ({+ label,+ value: yesNoUnknown(history[field]),+ help: 'Asked only when the last counted change of this kind sets back what the first changed.',+ })+ ),]: []),]}-export function buildSplitChanges(inputs: ExperimentHealthInputsApi | null): HealthDebugSplitChange[] {- return (inputs?.split_history?.changes ?? []).map((change) => ({- changedAt: formatTimestamp(change.changed_at),- before: describeSplit(change.before),- after: describeSplit(change.after),- counted: change.counted,- }))+const BUCKETING_SETTING_LABELS: Record<string, string> = {+ key: 'Key',+ aggregation: 'Match by group type',+ bucketing_identifier: 'Bucketing identifier',+ experience_continuity: 'Persist across authentication',+}++export function buildFlagChanges(inputs: ExperimentHealthInputsApi | null): HealthDebugFlagChange[] {+ const history = inputs?.flag_history+ if (!history) {+ return []+ }+ const changes = [+ ...history.split_changes.map((change) => ({+ at: change.changed_at,+ changed: 'Split',+ before: describeSplit(change.before),+ after: describeSplit(change.after),+ counted: change.counted,+ })),+ ...history.bucketing_changes.map((change) => ({+ at: change.changed_at,+ changed: BUCKETING_SETTING_LABELS[change.setting] ?? change.setting,+ before: change.before ?? 'none',+ after: change.after ?? 'none',+ counted: change.counted,+ })),+ ]+ return changes+ .sort((a, b) => new Date(a.at).getTime() - new Date(b.at).getTime())+ .map(({ at, ...change }) => ({ changedAt: formatTimestamp(at), ...change }))}const TRAFFIC_STATUS_LABELS: Record<ExperimentHealthInputsTrafficStatusEnumApi, string> = {diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex b1574c5037c5..2c2dcbbf673e 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -144,48 +144,50 @@ function flagStateCheck(input: HealthDebugInput): HealthDebugCheck {}}-function splitChangedNote(finding: ExperimentHealthFindingApi): string {- const { evidence } = finding- if (finding.subcode === 'not_checked') {- return evidence.change_count == null- ? `Not checked: ${evidence.reason}.`- : `Not checked: ${evidence.reason}. ${evidence.change_count} changes since launch, from ${evidence.from} to ${evidence.to}.`- }- const exposedBefore = evidence.exposed_before_change- return `${evidence.change_count} counted changes, from ${evidence.from} to ${evidence.to}. Exposed before the first: ${exposedBefore == null ? 'unknown' : exposedBefore ? 'yes' : 'no'}.`+function share(value: unknown): string | null {+ return typeof value === 'number' ? `${(value * 100).toFixed(1)}%` : null}-function splitChangedCheck(input: HealthDebugInput): HealthDebugCheck {- const finding = findingOf(input, 'split_changed_mid_run')+function serverOnlyCheck(+ input: HealthDebugInput,+ check: ExperimentHealthFindingApi['code'],+ note: ((finding: ExperimentHealthFindingApi) => string) | null+): HealthDebugCheck {+ const finding = findingOf(input, check)return {- check: 'split_changed_mid_run',+ check,source: 'health endpoint',reads: ['experiment', 'flag'],- ...serverResult('split_changed_mid_run', input.health, (code) => code === 'split_changed_mid_run'),+ ...serverResult(check, input.health, (code) => code === check),browser: null,browserLabel: BROWSER_RULES,...NO_TIMES,difference: null,- note: finding ? splitChangedNote(finding) : null,+ note: finding && note ? note(finding) : null,}}-function flagVariantsCheck(input: HealthDebugInput): HealthDebugCheck {- return {- check: 'flag_variants_mismatch',- source: 'health endpoint',- reads: ['experiment', 'flag'],- ...serverResult('flag_variants_mismatch', input.health, (code) => code === 'flag_variants_mismatch'),- browser: null,- browserLabel: BROWSER_RULES,- ...NO_TIMES,- difference: null,- note: null,+function exposedBeforeFirst({ evidence }: ExperimentHealthFindingApi): string {+ const exposedBefore = evidence.exposed_before_change+ return `Exposed before the first: ${exposedBefore == null ? 'unknown' : exposedBefore ? 'yes' : 'no'}.`+}++function splitChangedNote(finding: ExperimentHealthFindingApi): string {+ const { evidence } = finding+ if (finding.subcode === 'not_checked') {+ return evidence.change_count == null+ ? `Not checked: ${evidence.reason}.`+ : `Not checked: ${evidence.reason}. ${evidence.change_count} changes since launch, from ${evidence.from} to ${evidence.to}.`}+ return `${evidence.change_count} counted changes, from ${evidence.from} to ${evidence.to}. ${exposedBeforeFirst(finding)}`}-function share(value: unknown): string | null {- return typeof value === 'number' ? `${(value * 100).toFixed(1)}%` : null+function bucketingChangedNote(finding: ExperimentHealthFindingApi): string {+ const { evidence } = finding+ const changed = `${evidence.change_count} saves changed ${evidence.changed_settings}.`+ return finding.subcode === 'not_checked'+ ? `Not checked: ${evidence.reason}. ${changed}`+ : `${changed} ${exposedBeforeFirst(finding)}`}function forcedVariantNote(finding: ExperimentHealthFindingApi): string {@@ -208,25 +210,6 @@ function forcedVariantNote(finding: ExperimentHealthFindingApi): string {return parts.join(' ')}-function forcedVariantCheck(input: HealthDebugInput): HealthDebugCheck {- const finding = findingOf(input, 'forced_variant_release_condition')- return {- check: 'forced_variant_release_condition',- source: 'health endpoint',- reads: ['experiment', 'flag'],- ...serverResult(- 'forced_variant_release_condition',- input.health,- (code) => code === 'forced_variant_release_condition'- ),- browser: null,- browserLabel: BROWSER_RULES,- ...NO_TIMES,- difference: null,- note: finding ? forcedVariantNote(finding) : null,- }-}-function noMetricCheck(input: HealthDebugInput): HealthDebugCheck {const serverFires = !!findingOf(input, 'no_metric')const differs = !!input.health && input.browserNoMetricsWarning !== serverFires@@ -307,9 +290,10 @@ function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugCheck[] {return [flagStateCheck(input),- splitChangedCheck(input),- flagVariantsCheck(input),- forcedVariantCheck(input),+ serverOnlyCheck(input, 'split_changed_mid_run', splitChangedNote),+ serverOnlyCheck(input, 'bucketing_changed_mid_run', bucketingChangedNote),+ serverOnlyCheck(input, 'flag_variants_mismatch', null),+ serverOnlyCheck(input, 'forced_variant_release_condition', forcedVariantNote),noMetricCheck(input),...exposureChecks(input),]diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex f30bcbb3b743..d9e77759b50e 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -24,6 +24,7 @@ export type ExperimentHealthFindingCode =| 'forced_variant_release_condition'| 'flag_variants_mismatch'| 'split_changed_mid_run'+ | 'bucketing_changed_mid_run'export interface ExperimentHealthFinding {code: ExperimentHealthFindingCodediff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex d6a79528badd..962fde646094 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -44612,6 +44612,7 @@ export namespace Schemas {* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch* * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run*/export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];@@ -44628,6 +44629,7 @@ export namespace Schemas {ForcedVariantReleaseCondition: 'forced_variant_release_condition',FlagVariantsMismatch: 'flag_variants_mismatch',SplitChangedMidRun: 'split_changed_mid_run',+ BucketingChangedMidRun: 'bucketing_changed_mid_run',} as const;/**@@ -44686,7 +44688,8 @@ export namespace Schemas {* * `zero_exposures` - Zero Exposures* * `forced_variant_release_condition` - Forced Variant Release Condition* * `flag_variants_mismatch` - Flag Variants Mismatch- * * `split_changed_mid_run` - Split Changed Mid Run */+ * * `split_changed_mid_run` - Split Changed Mid Run+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run */code: ExperimentHealthFindingCodeEnum;/*** The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.@@ -45305,34 +45308,60 @@ export namespace Schemas {before: ExperimentHealthInputsVariant[];/** The split after the change. */after: ExperimentHealthInputsVariant[];- /** Whether the split-change check counts this change: it and every later change count from the first with an exposure before it, or an unknown answer. False for every change when not checked. */+ /** Whether the checks count this change: every change counts from the first change, of any kind, with an exposure before it, or an unknown answer. False for every change when not checked. */counted: boolean;}- export interface ExperimentHealthInputsSplitHistory {- /** Whether the split changes were compared with the exposures. False when not_checked_reason is set. */+ export interface ExperimentHealthInputsBucketingChange {+ /** When the setting changed. */+ changed_at: string;+ /** The flag setting that decides the variant draw: 'key', 'aggregation' (the group type the flag matches by), 'bucketing_identifier' or 'experience_continuity' (persist the flag across authentication steps). */+ setting: string;+ /**+ * The value before the change: the key, the group type index, 'distinct_id' or 'device_id', or 'true' or 'false'. Null for an aggregation by persons, or an unreadable key.+ * @nullable+ */+ before: string | null;+ /**+ * The value after the change, in the same form.+ * @nullable+ */+ after: string | null;+ /** Whether the checks count this change: every change counts from the first change, of any kind, with an exposure before it, or an unknown answer. False for every change when not checked. */+ counted: boolean;+ }++ export interface ExperimentHealthInputsFlagHistory {+ /** Whether the changes were compared with the exposures. False when not_checked_reason is set. */checked: boolean;/**- * Why the split changes were not compared with the exposures: 'no_history_access', 'no_query_access', 'exposure_filters', 'exposure_action' or 'exposure_criteria'. Null when they were.+ * Why the changes were not compared with the exposures: 'no_history_access', 'no_query_access', 'exposure_filters', 'exposure_action' or 'exposure_criteria'. Null when they were.* @nullable*/not_checked_reason: string | null;/** When the experiment launched. */launched_at: string;- /** The project's time zone, which the finding uses for times. */+ /** The project's time zone, which the findings use for times. */team_timezone: string;/** Every split change since launch. Empty for 'no_history_access', which does not read them. */- changes: ExperimentHealthInputsSplitChange[];+ split_changes: ExperimentHealthInputsSplitChange[];+ /** Every change since launch of a setting that decides the variant draw, one per setting and save. Empty for 'no_history_access'. */+ bucketing_changes: ExperimentHealthInputsBucketingChange[];/*** Whether anyone was exposed before the first counted change. Null when unknown or not checked.* @nullable*/exposed_before_first: boolean | null;/**- * Whether anyone was exposed between the first and the last change. Asked only when the last change restores the split from before the first. Null when not asked or unknown.+ * Whether anyone was exposed between the first and the last counted split change. Asked only when the last restores the split from before the first. Null when not asked or unknown.+ * @nullable+ */+ split_exposed_between: boolean | null;+ /**+ * Whether anyone was exposed between the first and the last counted bucketing change. Asked only when every setting was set back. Null when not asked or unknown.* @nullable*/- exposed_between: boolean | null;+ bucketing_exposed_between: boolean | null;}/**@@ -45376,9 +45405,9 @@ export namespace Schemas {}export interface ExperimentHealthInputsReaderAccess {- /** Whether the reader may read the flag's history, which the split-change check needs. */+ /** Whether the reader may read the flag's history, which the flag-change checks need. */can_read_flag_history: boolean;- /** Whether the reader may query events, which the split-change and forced-variant checks need. */+ /** Whether the reader may query events, which the flag-change and forced-variant checks need. */can_query_events: boolean;/** Event properties the reader is denied. The forced-variant check skips its scan when one of them is a flag call property it reads. */denied_event_properties: string[];@@ -45653,8 +45682,8 @@ export namespace Schemas {secondary_metric_count: number;/** The experiment's feature flag. Null when it has none. */flag: ExperimentHealthInputsFlag | null;- /** The changes of the flag's split since launch. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */- split_history: ExperimentHealthInputsSplitHistory | null;+ /** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */+ flag_history: ExperimentHealthInputsFlagHistory | null;/** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis.** * `read` - Read