Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): report release condition changes after launch (#119020)

Marcel Poelkerbb3674ac2d0e
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 9f2122a4b4f2..9a9d3e084c7c 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -10587,9 +10587,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.d5684a998e1dc7b37b7fe5db45432053b75fae05ca0538c87e0de6ce3c9dc8ca.2Qe4patz9tyjmFAEZsVCkjjqcrNyTbOAZ3aXHr2mg7g
+ hash: v1.k794b7964.89ba753555a9fc78822e75cf52e6783de7ea60c3046eed95150180b32ade9d41.P2cGRvwPp8IuU2bohVDvyURVapAxcTMO7RUmRE0WhjY
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.3f5dc6355429b4f0fb060a12d6cdbcee422a7a5d31b95fe77450bebe223b5bf8.F_WmHTt3NUbhkJF8UDfZZ_uET6EuHmF6mYZtGXMHBV4
+ hash: v1.k794b7964.879c2f4f4ad7a1d61402af628b5a578f0e88761c284a1e5bf7f7ef64aa438219.ehJIFQobltKAPwR0e0UXYsAY5pt95g_t-6kui8yerfM
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index bf5cd9fef201..680ae4216794 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9953,6 +9953,7 @@ export namespace Schemas {
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
*/
export type ExperimentHealthFindingCodeEnum =
| "flag_off_while_running"
@@ -9966,7 +9967,8 @@ export namespace Schemas {
| "forced_variant_release_condition"
| "flag_variants_mismatch"
| "split_changed_mid_run"
- | "bucketing_changed_mid_run";
+ | "bucketing_changed_mid_run"
+ | "release_conditions_changed_mid_run";
/**
* * `critical` - Critical severity
* * `warning` - Warning severity
@@ -10011,6 +10013,7 @@ export namespace Schemas {
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
*/
code: ExperimentHealthFindingCodeEnum;
/**
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 82160dc8d632..eaa622e1e671 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -110,6 +110,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
FLAG_VARIANTS_MISMATCH = "flag_variants_mismatch"
SPLIT_CHANGED_MID_RUN = "split_changed_mid_run"
BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"
+ RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"
class HealthExposureInputsStatus(LabeledStrEnum):
@@ -140,6 +141,7 @@ class ExperimentHealthFlagChangeKind(LabeledStrEnum):
SPLIT = "split"
# A setting that decides the variant draw: the key, the group type, the bucketing identifier or persistence.
BUCKETING = "bucketing"
+ RELEASE_CONDITIONS = "release_conditions", "Release-condition"
class ExperimentHealthFindingSeverity(LabeledStrEnum):
diff --git a/products/experiments/backend/health/checks/change_wording.py b/products/experiments/backend/health/checks/change_wording.py
index b35a2857eab4..4ab228fe1454 100644
--- a/products/experiments/backend/health/checks/change_wording.py
+++ b/products/experiments/backend/health/checks/change_wording.py
@@ -3,7 +3,11 @@
from datetime import datetime, timedelta
from typing import Literal
-from products.experiments.backend.health.context import FlagHistoryNotCheckedReason
+from products.experiments.backend.health.context import (
+ MAX_FLAG_SAVES,
+ MAX_RELEASE_CONDITIONS,
+ FlagHistoryNotCheckedReason,
+)
from products.experiments.backend.health.split_changes import format_change_time
SECONDS_PER_DAY = 24 * 60 * 60
@@ -47,6 +51,10 @@ def not_checked_because(reason: FlagHistoryNotCheckedReason) -> str:
return "this API key can't query events"
case FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS:
return "you can't view the linked feature flag's history"
+ case FlagHistoryNotCheckedReason.TOO_MANY_SAVES:
+ return f"the flag was saved more than {MAX_FLAG_SAVES} times since launch"
+ case FlagHistoryNotCheckedReason.TOO_MANY_CONDITIONS:
+ return f"the flag has more than {MAX_RELEASE_CONDITIONS} release conditions"
def not_checked_next_step(reason: FlagHistoryNotCheckedReason, the_change: str) -> str:
diff --git a/products/experiments/backend/health/checks/conditions_changed.py b/products/experiments/backend/health/checks/conditions_changed.py
new file mode 100644
index 000000000000..ac4530b50abf
--- /dev/null
+++ b/products/experiments/backend/health/checks/conditions_changed.py
@@ -0,0 +1,356 @@
+"""
+The release conditions of the experiment's flag changed after launch while people were already exposed. Pure
+functions, no I/O: `health/exposure_gate.py` loads the changes and the exposure answers.
+
+A condition added, removed or with other criteria moves people into or out of the experiment. The people exposed
+before stay in the results, so the results mix two audiences. That is information, so it is graded info. A new
+order of conditions that serve different variants moves the people who match several of them to another variant.
+How many is unknown, so that is info too.
+
+A variant override that was removed moves the people its condition matches from its variant to a random one, and
+their exposures under that variant stay in the results. It is graded by the share of that variant's users the
+condition serves, as the forced-variant check grades an override in place. An override that is still in place is
+the forced-variant check's to report, so it is not reported here.
+"""
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthFlagChangeKind,
+)
+from products.experiments.backend.health.checks.change_wording import suggested_start, the_change
+from products.experiments.backend.health.checks.flag_change_findings import (
+ change_finding,
+ changes_not_checked,
+ history_to_check,
+ not_checked_finding,
+)
+from products.experiments.backend.health.condition_changes import (
+ ConditionsChangeShape,
+ ConditionsNetEffect,
+ RemovedOverride,
+ conditions_net_effect,
+)
+from products.experiments.backend.health.context import (
+ ConditionsChange,
+ FlagHistory,
+ FlagHistoryNotChecked,
+ HealthContext,
+ ReachableCondition,
+)
+from products.experiments.backend.health.pinned_share import (
+ PinnedConditions,
+ PinnedShareGrade,
+ PinnedShareUnknownReason,
+ grade_pinned_share,
+ percent,
+ pinned_conditions,
+ pinned_share_severity,
+)
+from products.experiments.backend.health.split_changes import format_change_time
+
+_TITLE = "The release conditions changed after launch"
+_OVERRIDE_TITLE = "A variant override was removed after launch"
+_REORDERED_TITLE = "The order of the release conditions changed after launch"
+_REVERTED_TITLE = "The release conditions changed and were set back"
+_NOT_CHECKED_TITLE = "Release condition changes were not checked"
+
+
+def release_conditions_changed_mid_run(ctx: HealthContext) -> ExperimentHealthFinding | None:
+ history = history_to_check(ctx)
+ if history is None:
+ return None
+ if isinstance(history, FlagHistoryNotChecked):
+ return _not_checked(history)
+ effect = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
+ if effect is None:
+ return None
+ # Conditions set back with nobody exposed in between left no trace in the results.
+ if (
+ effect.shape == ConditionsChangeShape.REVERTED
+ and history.exposed_between.get(ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS) is False
+ ):
+ return None
+ removed = _reported(ctx, history, effect)
+ if removed:
+ finding = _override_removed(ctx, history, effect, removed)
+ if finding is not None:
+ return finding
+ change = the_change(effect.change_count, "the")
+ if effect.reordered:
+ return _finding(
+ history,
+ effect,
+ subcode=ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ title=_REORDERED_TITLE,
+ sentences=[
+ f"{_changed(history, effect)}{_condition_counts(effect)}, and conditions that serve different "
+ "variants now come in another order.",
+ "Users who match more than one of them now get the variant of another condition, so those exposed "
+ f"before {change} may have switched variants. Their exposures stay in the results.",
+ f"Review {change} in the flag history.",
+ ],
+ diagnostic_ref="A7",
+ )
+ if effect.audience_changed:
+ return _finding(
+ history,
+ effect,
+ subcode=ConditionsChangeShape.CONDITIONS_CHANGED,
+ title=_TITLE,
+ sentences=[
+ f"{_changed(history, effect)}{_condition_counts(effect)}.",
+ f"People who match differently now enter or leave the experiment, and people exposed before {change} "
+ "stay in the results.",
+ *_edited_overrides(effect),
+ f"Review {change} in the flag history.",
+ ],
+ )
+ # Only an override changed. The forced-variant check reports one that is still in place, and the removed ones
+ # were graded above.
+ if effect.shape != ConditionsChangeShape.REVERTED or removed:
+ return None
+ return _finding(
+ history,
+ effect,
+ subcode=ConditionsChangeShape.REVERTED,
+ title=_REVERTED_TITLE,
+ sentences=[
+ _set_back(history, effect),
+ "People exposed in between were targeted by other conditions, and they stay in the results.",
+ f"Review {change} in the flag history.",
+ ],
+ )
+
+
+def _edited_overrides(effect: ConditionsNetEffect) -> list[str]:
+ edited = effect.edited_overrides
+ if not edited:
+ return []
+ if len(edited) == 1:
+ return [
+ f'Set {edited[0].condition_number} has the variant override "{edited[0].variant}", so users who now match '
+ "it or no longer match it may also have switched variants."
+ ]
+ numbers = [override.condition_number for override in edited]
+ names = ", ".join(str(number) for number in numbers[:-1]) + f" and {numbers[-1]}"
+ return [
+ f"Sets {names} have variant overrides, so users who now match them or no longer match them may also have "
+ "switched variants."
+ ]
+
+
+def removed_overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
+ """The overrides removed after launch, as pins on the conditions that held them, which the finding is graded
+ by as if they were still in place. None when no removed override has a condition left to count its users."""
+ history = ctx.flag_history
+ flag = ctx.flag
+ if not isinstance(history, FlagHistory) or flag is None or flag.deleted or not ctx.is_running or ctx.archived:
+ return None
+ effect = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
+ if effect is None:
+ return None
+ removed = _reported(ctx, history, effect)
+ pinned = tuple(
+ ReachableCondition(number=override.condition_number, pinned_variant=override.variant)
+ for override in removed
+ if override.condition_number is not None and override.same_audience
+ )
+ if not pinned:
+ return None
+ return pinned_conditions(pinned, flag.variants, all_counted=len(pinned) == len(removed))
+
+
+def _reported(ctx: HealthContext, history: FlagHistory, effect: ConditionsNetEffect) -> list[RemovedOverride]:
+ # An excluded variant is not in the results, so an override to it biased nothing. An override that changed and
+ # changed back while nobody was exposed moved nobody.
+ return [
+ override
+ for override in effect.overrides.removed
+ if override.variant not in ctx.excluded_variant_keys
+ and ((window := override.temporary_window) is None or history.exposed_during_overrides.get(window) is not False)
+ ]
+
+
+def _override_removed(
+ ctx: HealthContext, history: FlagHistory, effect: ConditionsNetEffect, removed: list[RemovedOverride]
+) -> ExperimentHealthFinding | None:
+ pins = removed_overrides_to_grade(ctx)
+ grade = grade_pinned_share(pins, ctx.condition_set_traffic) if pins is not None and pins.weighted else None
+ severity = pinned_share_severity(grade)
+ if not removed:
+ return None
+ if severity is None:
+ # A pin to a variant with 0% in the split was the only source of that variant's results, however few.
+ if pins is None or not pins.unweighted:
+ return None
+ severity = ExperimentHealthFindingSeverity.INFO
+ last_changed_at = max(override.restored_at or override.removed_at for override in removed)
+ return _finding(
+ history,
+ effect,
+ subcode=ConditionsChangeShape.REVERTED
+ if effect.shape == ConditionsChangeShape.REVERTED
+ else ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ title=_OVERRIDE_TITLE,
+ severity=severity,
+ sentences=[
+ *(_removed(override, history.team_timezone) for override in removed),
+ *_moved(removed),
+ *_measured(grade, removed),
+ f"Reset the experiment, or move its start date to "
+ f"{suggested_start(last_changed_at, history.team_timezone)}.",
+ ],
+ grade=grade,
+ diagnostic_ref="A7",
+ )
+
+
+def _moved(removed: list[RemovedOverride]) -> list[str]:
+ if len(removed) == 1 and removed[0].restored_at is not None:
+ return [
+ f'Between these changes, users it matches did not get "{removed[0].variant}", so those exposed then may '
+ "have switched variants. Their exposures stay in the results."
+ ]
+ if any(override.restored_at is not None for override in removed):
+ return [
+ "For a while, users they match did not get the override's variant, so those exposed then may have "
+ "switched variants. Their exposures stay in the results."
+ ]
+ if len(removed) == 1:
+ variant = f'"{removed[0].variant}"'
+ return [
+ f"Users it matched got {variant} while the override was in place and a random variant after, so those "
+ f"exposed while it was in place may have switched variants. Their exposures under {variant} stay in the "
+ "results."
+ ]
+ return [
+ "Users they matched got the override's variant while it was in place and a random variant after, so those "
+ "exposed while it was in place may have switched variants. Their exposures stay in the results."
+ ]
+
+
+def _removed(override: RemovedOverride, team_timezone: str) -> str:
+ removed_at = format_change_time(override.removed_at, team_timezone)
+ if override.restored_at is not None:
+ restored_at = format_change_time(override.restored_at, team_timezone)
+ return (
+ f'The variant override "{override.variant}" of Set {override.condition_number} changed on {removed_at} '
+ f"and was set back on {restored_at}."
+ )
+ set_at = format_change_time(override.set_at, team_timezone) if override.set_at is not None else None
+ if override.condition_number is None:
+ subject = f'A release condition with the variant override "{override.variant}"'
+ set_verb = "added"
+ else:
+ subject = f'The variant override "{override.variant}" of Set {override.condition_number}'
+ set_verb = "set"
+ if set_at is None:
+ return f"{subject} was removed on {removed_at}."
+ return f"{subject} was {set_verb} on {set_at} and removed on {removed_at}."
+
+
+def _measured(grade: PinnedShareGrade | None, removed: list[RemovedOverride]) -> list[str]:
+ single = len(removed) == 1 and removed[0].condition_number is not None
+ subject = f"Set {removed[0].condition_number}" if single else "these conditions"
+ matches = "matches" if single else "match"
+ measurement = grade.measurement if grade is not None else None
+ if grade is not None and grade.share is not None and measurement is not None:
+ since = format_change_time(measurement.since, measurement.team_timezone)
+ return [
+ f"Since {since}, {subject} matched about {percent(grade.share)} of the users who would get "
+ f'"{measurement.most_pinned_variant}" with the override in place ({measurement.pinned_people:,} of '
+ f"about {round(measurement.variant_people):,})."
+ ]
+ if grade is None:
+ return [
+ f"PostHog can't measure how many users {'the override' if len(removed) == 1 else 'the overrides'} matched."
+ ]
+ if grade.unknown_reason == PinnedShareUnknownReason.EVALUATED_OUTSIDE_FLAG_SERVICE:
+ return [
+ "Most of this flag's calls are evaluated locally in a server SDK, bootstrapped, or assigned in your own "
+ f"code, so PostHog can't measure how many users {subject} {matches}."
+ ]
+ if grade.unknown_reason == PinnedShareUnknownReason.TOO_FEW_PEOPLE and measurement is not None:
+ since = format_change_time(measurement.since, measurement.team_timezone)
+ return [f"Too few users called the flag since {since} to measure how many users {subject} {matches}."]
+ return [f"PostHog can't measure how many users {subject} {matches}."]
+
+
+def _finding(
+ history: FlagHistory,
+ effect: ConditionsNetEffect,
+ *,
+ subcode: ConditionsChangeShape,
+ title: str,
+ sentences: list[str],
+ severity: ExperimentHealthFindingSeverity = ExperimentHealthFindingSeverity.INFO,
+ grade: PinnedShareGrade | None = None,
+ diagnostic_ref: str = "A6",
+) -> ExperimentHealthFinding:
+ return change_finding(
+ history,
+ effect,
+ code=ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN,
+ subcode=subcode.value,
+ severity=severity,
+ title=title,
+ sentences=sentences,
+ kind_evidence={**_kind_evidence(effect), **(grade.evidence() if grade is not None else {})},
+ diagnostic_ref=diagnostic_ref,
+ )
+
+
+def _not_checked(history: FlagHistoryNotChecked) -> ExperimentHealthFinding | None:
+ effect = conditions_net_effect(changes_not_checked(history, ConditionsChange), current=None)
+ if effect is None:
+ return None
+ return not_checked_finding(
+ history,
+ effect,
+ code=ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN,
+ title=_NOT_CHECKED_TITLE,
+ what_changed=[f"{_changed(history, effect)}{_condition_counts(effect)}."],
+ kind_evidence=_kind_evidence(effect),
+ diagnostic_ref=_diagnostic_ref(effect),
+ )
+
+
+def _changed(history: FlagHistory | FlagHistoryNotChecked, effect: ConditionsNetEffect) -> str:
+ first_at = format_change_time(effect.first.changed_at, history.team_timezone)
+ if effect.change_count == 1:
+ return f"The release conditions changed on {first_at}"
+ last_at = format_change_time(effect.last.changed_at, history.team_timezone)
+ return f"The release conditions changed {effect.change_count} times between {first_at} and {last_at}"
+
+
+def _set_back(history: FlagHistory, effect: ConditionsNetEffect) -> str:
+ first_at = format_change_time(effect.first.changed_at, history.team_timezone)
+ last_at = format_change_time(effect.last.changed_at, history.team_timezone)
+ if effect.change_count == 2:
+ return f"The release conditions changed on {first_at} and were set back on {last_at}."
+ return (
+ f"The release conditions changed {effect.change_count} times between {first_at} and {last_at}, and the "
+ "last change set them back."
+ )
+
+
+def _condition_counts(effect: ConditionsNetEffect) -> str:
+ before = len(effect.from_conditions)
+ after = len(effect.to_conditions)
+ if before == after:
+ return ""
+ return f", from {before} {_conditions(before)} to {after} {_conditions(after)}"
+
+
+def _conditions(count: int) -> str:
+ return "condition" if count == 1 else "conditions"
+
+
+def _diagnostic_ref(effect: ConditionsNetEffect) -> str:
+ return "A7" if effect.shape == ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED else "A6"
+
+
+def _kind_evidence(effect: ConditionsNetEffect) -> dict[str, int]:
+ return {"conditions_before": len(effect.from_conditions), "conditions_after": len(effect.to_conditions)}
diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.py
index f4e60455655f..30df574e5156 100644
--- a/products/experiments/backend/health/checks/flag_change_findings.py
+++ b/products/experiments/backend/health/checks/flag_change_findings.py
@@ -31,6 +31,17 @@ from products.experiments.backend.health.context import (
# `finding_variant` on the health finding events.
NOT_CHECKED = "not_checked"
+EvidenceValue = str | int | float | bool | None
+
+# The reasons for which the flag history is not read, so no change is known.
+_HISTORY_NOT_READ = frozenset(
+ {
+ FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS,
+ FlagHistoryNotCheckedReason.TOO_MANY_SAVES,
+ FlagHistoryNotCheckedReason.TOO_MANY_CONDITIONS,
+ }
+)
+
class _Change(Protocol):
@property
@@ -63,13 +74,11 @@ def history_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked
def flag_history_not_readable(ctx: HealthContext) -> ExperimentHealthFinding | None:
- """One finding for every kind of change when the reader can't view the flag's history."""
+ """One finding for every kind of change when the flag's history is not read."""
history = history_to_check(ctx)
- if (
- not isinstance(history, FlagHistoryNotChecked)
- or history.reason != FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS
- ):
+ if not isinstance(history, FlagHistoryNotChecked) or history.reason not in _HISTORY_NOT_READ:
return None
+ no_access = history.reason == FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS
return ExperimentHealthFinding(
# pinned: insights and the page read this code and subcode for this finding.
code=ExperimentHealthFindingCode.SPLIT_CHANGED_MID_RUN,
@@ -77,21 +86,25 @@ def flag_history_not_readable(ctx: HealthContext) -> ExperimentHealthFinding | N
severity=ExperimentHealthFindingSeverity.INFO,
title="The flag history was not checked",
detail=(
- "This check can't see whether the flag's variant split, key or bucketing changed after launch, "
- f"because {not_checked_because(history.reason)}. Viewing the history needs viewer access to the "
- "flag and, with an API key, the activity_log:read scope."
+ "This check can't see whether the flag's variant split, key, bucketing or release conditions "
+ f"changed after launch, because {not_checked_because(history.reason)}. "
+ + (
+ "Viewing the history needs viewer access to the flag and, with an API key, the activity_log:read scope."
+ if no_access
+ else "Review the flag history."
+ )
),
evidence={"reason": history.reason.value},
# The flag history is the page this caller can't open.
- actions=(),
+ actions=() if no_access else (ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),
diagnostic_ref="E3",
)
def changes_not_checked[C: FlagChange](history: FlagHistoryNotChecked, kind: type[C]) -> tuple[C, ...]:
- """The changes of one kind that a check reports as not checked. Empty for a history the reader can't view, which
+ """The changes of one kind that a check reports as not checked. Empty for a history that was not read, which
`flag_history_not_readable` reports."""
- if history.reason == FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS:
+ if history.reason in _HISTORY_NOT_READ:
return ()
return history.of_kind(kind)
@@ -105,7 +118,7 @@ def change_finding(
severity: ExperimentHealthFindingSeverity,
title: str,
sentences: Sequence[str],
- kind_evidence: Mapping[str, str | int | bool | None],
+ kind_evidence: Mapping[str, EvidenceValue],
diagnostic_ref: str,
) -> ExperimentHealthFinding:
return ExperimentHealthFinding(
@@ -129,7 +142,7 @@ def not_checked_finding(
code: ExperimentHealthFindingCode,
title: str,
what_changed: Sequence[str],
- kind_evidence: Mapping[str, str | int | bool | None],
+ kind_evidence: Mapping[str, EvidenceValue],
diagnostic_ref: str,
) -> ExperimentHealthFinding:
change = the_change(effect.change_count, "the")
@@ -159,9 +172,9 @@ def change_evidence(
history: FlagHistory | FlagHistoryNotChecked,
effect: ChangeEffect,
*,
- kind_evidence: Mapping[str, str | int | bool | None],
+ kind_evidence: Mapping[str, EvidenceValue],
exposed_before_change: bool | None,
-) -> dict[str, str | int | bool | None]:
+) -> dict[str, EvidenceValue]:
return {
"change_count": effect.change_count,
"first_changed_at": effect.first.changed_at.astimezone(UTC).isoformat(),
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index 1fbbe7fd6a22..3075afa0743e 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -8,8 +8,7 @@ See `get_match` in rust/feature-flags/src/flags/flag_matching.rs and `pinned_var
"""
from collections.abc import Mapping, Sequence
-from datetime import datetime, timedelta
-from enum import StrEnum
+from datetime import UTC
from posthog.dataclasses import frozen
@@ -20,7 +19,16 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.checks.flag_state import is_single_variant_shipped, shipped_variant_key
-from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext, ReachableCondition, Split
+from products.experiments.backend.health.condition_changes import conditions_net_effect
+from products.experiments.backend.health.context import ConditionsChange, FlagHistory, HealthContext, ReachableCondition
+from products.experiments.backend.health.pinned_share import (
+ PinnedConditions,
+ PinnedShareGrade,
+ grade_pinned_share,
+ pinned_conditions,
+ pinned_share_severity,
+ pinned_variant_keys,
+)
from products.experiments.backend.health.split_changes import format_change_time
# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.
@@ -31,28 +39,6 @@ ALL_CONDITIONS_PINNED = "all_conditions_pinned"
# assignment, so they bias the comparison unless the analysis excludes them, for example as test users.
SOME_CONDITIONS_PINNED = "some_conditions_pinned"
-# The severity of SOME_CONDITIONS_PINNED follows the largest share that pinned users make up of one variant's users,
-# in this window, or since launch or the flag's last save if later. The bias is per variant, so a few pinned users
-# weigh more in a small variant. Below the first share they can't move a variant's results much, so there is no
-# finding. Above the second they are a warning.
-TRAFFIC_WINDOW = timedelta(days=7)
-_NO_FINDING_BELOW_SHARE = 0.01
-_WARNING_ABOVE_SHARE = 0.05
-# With fewer people, the share of a few pinned people is mostly chance.
-_MIN_MATCHED_PEOPLE = 100
-# Only calls that the flag service evaluated name a condition set. When fewer calls than this share did, the
-# pinned users can be among the calls that name none, so a share under the first threshold does not clear the
-# finding. A share above it still counts, because the pinned users it found are real.
-_MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR = 0.5
-
-
-class PinnedShareUnknownReason(StrEnum):
- # pinned: the finding's `evidence.reason`, which the health finding events track.
- NOT_MEASURED = "not_measured"
- TOO_FEW_PEOPLE = "too_few_people"
- # Most calls carry no version: local evaluation in a server SDK, a bootstrap, or the customer's own assignment.
- EVALUATED_OUTSIDE_FLAG_SERVICE = "evaluated_outside_flag_service"
-
@frozen
class _Pins:
@@ -60,86 +46,15 @@ class _Pins:
all_pinned: bool
# The pinned conditions whose variant the analysis includes, in evaluation order.
analyzed: tuple[ReachableCondition, ...]
- weighted: tuple[ReachableCondition, ...]
- split: Split
- # The analyzed pins to a variant with 0% in the split. The split sends nobody there, so they don't bias the
- # other variants, but that variant's results show only the pinned users.
- unweighted: tuple[ReachableCondition, ...]
-
-
-@frozen
-class _Measurement:
- most_pinned_variant: str
- pinned_people: int
- # The variant's pinned people, and its weight in the split of the people assigned at random.
- variant_people: float
- # The people that a condition set served, without those pinned to a variant with 0% in the split.
- matched_people: int
- remotely_evaluated_share: float | None
- since: datetime
- team_timezone: str
-
- @property
- def share(self) -> float:
- return self.pinned_people / self.variant_people if self.variant_people else 0.0
-
-
-@frozen
-class _Grade:
- """The share that sets the severity of SOME_CONDITIONS_PINNED, or the reason it is unknown."""
-
- measurement: _Measurement | None
- share: float | None
- unknown_reason: PinnedShareUnknownReason | None
-
- def evidence(self) -> dict[str, str | int | float | None]:
- evidence: dict[str, str | int | float | None] = {}
- if self.measurement is not None:
- remotely_evaluated_share = self.measurement.remotely_evaluated_share
- evidence["most_pinned_variant"] = self.measurement.most_pinned_variant
- evidence["pinned_people"] = self.measurement.pinned_people
- evidence["variant_people"] = round(self.measurement.variant_people)
- evidence["matched_people"] = self.measurement.matched_people
- evidence["remotely_evaluated_share"] = (
- None if remotely_evaluated_share is None else round(remotely_evaluated_share, 4)
- )
- if self.share is not None:
- evidence["pinned_share"] = round(self.share, 4)
- if self.unknown_reason is not None:
- evidence["reason"] = self.unknown_reason.value
- return evidence
-
- def sentence(self, them: str) -> str:
- """The measured share in words, or why it can't be measured, after a leading space. Empty otherwise."""
- measurement = self.measurement
- if self.share is not None and measurement is not None:
- return (
- f" Since {format_change_time(measurement.since, measurement.team_timezone)}, about "
- f'{_percent(self.share)} of the users who got "{measurement.most_pinned_variant}" matched {them} '
- f"({measurement.pinned_people:,} of about {round(measurement.variant_people):,})."
- )
- if self.unknown_reason == PinnedShareUnknownReason.EVALUATED_OUTSIDE_FLAG_SERVICE:
- return (
- " Most of this flag's calls are evaluated locally in a server SDK, bootstrapped, or assigned in your "
- f"own code, so PostHog can't measure how many users match {them}."
- )
- return ""
-
-
-def needs_condition_set_traffic(ctx: HealthContext) -> bool:
- """Whether the check grades SOME_CONDITIONS_PINNED by the users the pinned conditions served."""
- pins = _pins(ctx)
- return ctx.is_launched and pins is not None and not pins.all_pinned and bool(pins.weighted)
+ graded: PinnedConditions
-def has_too_few_people(ctx: HealthContext, traffic: ConditionSetTraffic) -> bool:
- """Whether the check leaves the share of `traffic` unknown only because too few people count."""
+def overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
+ """The overrides in place that SOME_CONDITIONS_PINNED is graded by, or None when it is not graded."""
pins = _pins(ctx)
- return (
- pins is not None
- and bool(pins.weighted)
- and _grade(pins, traffic).unknown_reason == PinnedShareUnknownReason.TOO_FEW_PEOPLE
- )
+ if not ctx.is_launched or pins is None or pins.all_pinned:
+ return None
+ return pins.graded
def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFinding | None:
@@ -150,21 +65,23 @@ def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFind
is_draft = not ctx.is_launched
if not pins.all_pinned:
- return _some_conditions_pinned(pins, ctx.condition_set_traffic, is_draft)
+ return _some_conditions_pinned(pins, ctx, is_draft)
pinned_numbers = [condition.number for condition in pins.pinned]
- pinned_keys = _variant_keys(pins.pinned)
+ pinned_keys = pinned_variant_keys(pins.pinned)
# The shipped finding already says that everyone gets this variant.
if ctx.is_running and is_single_variant_shipped(flag) and pinned_keys == [shipped_variant_key(flag)]:
return None
+ set_after_launch = _set_after_launch(ctx, pinned_numbers)
return _finding(
subcode=ALL_CONDITIONS_PINNED,
severity=ExperimentHealthFindingSeverity.WARNING,
title="No users are randomly assigned to a variant",
- detail=_all_pinned_detail(pinned_numbers, pinned_keys, is_draft),
+ detail=_all_pinned_detail(pinned_numbers, pinned_keys, is_draft, set_after_launch),
pinned_numbers=pinned_numbers,
pinned_keys=pinned_keys,
diagnostic_ref="A7b",
+ grade_evidence=set_after_launch.evidence,
)
@@ -178,98 +95,89 @@ def _pins(ctx: HealthContext) -> _Pins | None:
return None
# A variant that the experiment excludes is not in the results, so its pins can't bias them.
analyzed = tuple(condition for condition in pinned if condition.pinned_variant not in ctx.excluded_variant_keys)
- unweighted_keys = {variant.key for variant in flag.variants if variant.rollout_percentage == 0}
return _Pins(
pinned=pinned,
all_pinned=len(pinned) == len(flag.reachable_conditions),
analyzed=analyzed,
- weighted=tuple(condition for condition in analyzed if condition.pinned_variant not in unweighted_keys),
- split=flag.variants,
- unweighted=tuple(condition for condition in analyzed if condition.pinned_variant in unweighted_keys),
+ graded=pinned_conditions(analyzed, flag.variants),
)
-def _some_conditions_pinned(
- pins: _Pins, traffic: ConditionSetTraffic | None, is_draft: bool
-) -> ExperimentHealthFinding | None:
+def _some_conditions_pinned(pins: _Pins, ctx: HealthContext, is_draft: bool) -> ExperimentHealthFinding | None:
if not pins.analyzed:
return None
- grade = None if is_draft or not pins.weighted else _grade(pins, traffic)
- share = grade.share if grade is not None else None
- if share is not None and share < _NO_FINDING_BELOW_SHARE and not pins.unweighted:
- return None
+ graded = pins.graded
+ grade = None if is_draft or not graded.weighted else grade_pinned_share(graded, ctx.condition_set_traffic)
+ severity = pinned_share_severity(grade)
+ if severity is None:
+ # A pin to a variant with 0% in the split is the only source of that variant's results, however few.
+ if not graded.unweighted:
+ return None
+ severity = ExperimentHealthFindingSeverity.INFO
pinned_numbers = [condition.number for condition in pins.analyzed]
- pinned_keys = _variant_keys(pins.analyzed)
+ pinned_keys = pinned_variant_keys(pins.analyzed)
+ set_after_launch = _set_after_launch(ctx, pinned_numbers)
return _finding(
subcode=SOME_CONDITIONS_PINNED,
- severity=ExperimentHealthFindingSeverity.WARNING
- if share is not None and share > _WARNING_ABOVE_SHARE
- else ExperimentHealthFindingSeverity.INFO,
+ severity=severity,
title="Some users are not randomly assigned to a variant",
- detail=_some_pinned_detail(pinned_numbers, pinned_keys, is_draft, grade, _variant_keys(pins.unweighted)),
+ detail=_some_pinned_detail(
+ pinned_numbers, pinned_keys, is_draft, grade, pinned_variant_keys(graded.unweighted), set_after_launch
+ ),
pinned_numbers=pinned_numbers,
pinned_keys=pinned_keys,
diagnostic_ref="A7",
- grade_evidence=grade.evidence() if grade is not None else {},
+ grade_evidence={**(grade.evidence() if grade is not None else {}), **set_after_launch.evidence},
)
-def _grade(pins: _Pins, traffic: ConditionSetTraffic | None) -> _Grade:
- if traffic is None:
- return _Grade(measurement=None, share=None, unknown_reason=PinnedShareUnknownReason.NOT_MEASURED)
- measurement = _measure(pins, traffic)
- remotely_evaluated_share = measurement.remotely_evaluated_share
- unknown_reason = (
- PinnedShareUnknownReason.EVALUATED_OUTSIDE_FLAG_SERVICE
- if remotely_evaluated_share is not None and remotely_evaluated_share < _MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR
- else PinnedShareUnknownReason.TOO_FEW_PEOPLE
- )
- if measurement.matched_people < _MIN_MATCHED_PEOPLE:
- return _Grade(measurement=measurement, share=None, unknown_reason=unknown_reason)
- share = measurement.share
- clearing_needs_more_calls = (
- remotely_evaluated_share is None or remotely_evaluated_share < _MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR
+@frozen
+class _SetAfterLaunch:
+ # One sentence after a leading space, or empty when no override was set after launch.
+ sentence: str
+ evidence: dict[str, str | int | float | None]
+ # Whether every override of these conditions was set after launch. Removing them then puts back the variants
+ # that users got at launch, so moving the start date is enough.
+ covers_all: bool
+
+ def remedy_after_removal(self) -> str:
+ if self.covers_all:
+ return "Then move the experiment's start date to after the removal."
+ return "Then reset the experiment, or move its start date to after the removal."
+
+
+def _set_after_launch(ctx: HealthContext, numbers: Sequence[int]) -> _SetAfterLaunch:
+ """The overrides of these conditions that were set after launch while people were already exposed. The
+ release-condition check leaves them to this finding, so that one override gets one row."""
+ history = ctx.flag_history
+ effect = (
+ conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
+ if isinstance(history, FlagHistory)
+ else None
)
- if share < _NO_FINDING_BELOW_SHARE and clearing_needs_more_calls:
- return _Grade(measurement=measurement, share=None, unknown_reason=unknown_reason)
- return _Grade(measurement=measurement, share=share, unknown_reason=None)
-
-
-def _measure(pins: _Pins, traffic: ConditionSetTraffic) -> _Measurement:
- excluded_numbers = {condition.number for condition in pins.unweighted}
- matched_people = sum(people for number, people in traffic.people_by_set.items() if number not in excluded_numbers)
- pinned_by_variant = dict.fromkeys(_variant_keys(pins.weighted), 0)
- for condition in pins.weighted:
- if condition.pinned_variant is not None:
- pinned_by_variant[condition.pinned_variant] += traffic.people_by_set.get(condition.number, 0)
- randomized_people = matched_people - sum(pinned_by_variant.values())
- weights = {variant.key: variant.rollout_percentage for variant in pins.split}
-
- def variant_people(key: str) -> float:
- weight = weights.get(key)
- # An unreadable weight counts as the whole split, which gives the lowest share the pins can have.
- return pinned_by_variant[key] + randomized_people * (1.0 if weight is None else weight / 100)
-
- def share(key: str) -> float:
- people = variant_people(key)
- return pinned_by_variant[key] / people if people else 0.0
-
- most_pinned_variant = max(pinned_by_variant, key=share)
- return _Measurement(
- most_pinned_variant=most_pinned_variant,
- pinned_people=pinned_by_variant[most_pinned_variant],
- variant_people=variant_people(most_pinned_variant),
- matched_people=matched_people,
- remotely_evaluated_share=traffic.remotely_evaluated_calls / traffic.calls if traffic.calls else None,
- since=traffic.since,
- team_timezone=traffic.team_timezone,
+ added = [] if effect is None else [o for o in effect.overrides.added if o.condition_number in numbers]
+ if history is None or not added:
+ return _SetAfterLaunch(sentence="", evidence={}, covers_all=False)
+ set_at = max(override.set_at for override in added)
+ at = format_change_time(set_at, history.team_timezone)
+ if len(added) == 1:
+ sentence = (
+ f" The override of Set {added[0].condition_number} was set on {at}, after launch, so users it matches "
+ "who were exposed before may have switched variants."
+ )
+ else:
+ names = _set_names(sorted(override.condition_number for override in added))
+ sentence = (
+ f" The overrides of {names} were set after launch, last on {at}, so users they match who were exposed "
+ "before may have switched variants."
+ )
+ return _SetAfterLaunch(
+ sentence=sentence,
+ evidence={"override_set_at": set_at.astimezone(UTC).isoformat()},
+ covers_all={override.condition_number for override in added} >= set(numbers),
)
-def _variant_keys(conditions: Sequence[ReachableCondition]) -> list[str]:
- return list(dict.fromkeys(condition.pinned_variant for condition in conditions if condition.pinned_variant))
-
-
def _quoted(keys: Sequence[str]) -> str:
return ", ".join(f'"{key}"' for key in keys)
@@ -280,7 +188,9 @@ def _set_names(numbers: Sequence[int]) -> str:
return "Sets " + ", ".join(str(number) for number in numbers[:-1]) + f" and {numbers[-1]}"
-def _all_pinned_detail(numbers: Sequence[int], pinned_keys: Sequence[str], is_draft: bool) -> str:
+def _all_pinned_detail(
+ numbers: Sequence[int], pinned_keys: Sequence[str], is_draft: bool, set_after_launch: _SetAfterLaunch
+) -> str:
overrides = "the override" if len(numbers) == 1 else "the overrides"
when = "after launch " if is_draft else ""
if len(pinned_keys) == 1:
@@ -292,7 +202,8 @@ def _all_pinned_detail(numbers: Sequence[int], pinned_keys: Sequence[str], is_dr
why = f'With no other variant to compare against, the results can\'t show whether "{key}" makes a difference.'
fix = (
f'If you meant to roll out "{key}", end the experiment with a conclusion. Otherwise remove {overrides} '
- f'so users are assigned at random. Users who already saw "{key}" can then switch to another variant.'
+ f'so users are assigned at random. Users who already saw "{key}" can then switch to another variant. '
+ f"{set_after_launch.remedy_after_removal()}"
)
else:
fact = (
@@ -305,19 +216,20 @@ def _all_pinned_detail(numbers: Sequence[int], pinned_keys: Sequence[str], is_dr
)
fix = (
f"Remove {overrides} so users are assigned at random. Users who already saw a variant can then "
- "switch to another one."
+ f"switch to another one. {set_after_launch.remedy_after_removal()}"
)
if is_draft:
fix = f"Remove {overrides} before launch so users are assigned at random."
- return f"{fact} {why} {fix}"
+ return f"{fact}{set_after_launch.sentence} {why} {fix}"
def _some_pinned_detail(
numbers: Sequence[int],
pinned_keys: Sequence[str],
is_draft: bool,
- grade: _Grade | None,
+ grade: PinnedShareGrade | None,
unweighted_keys: Sequence[str],
+ set_after_launch: _SetAfterLaunch,
) -> str:
if len(numbers) == 1:
key = pinned_keys[0]
@@ -344,7 +256,8 @@ def _some_pinned_detail(
else:
fix = (
"If these are internal or test users, check that the filter for internal and test users excludes "
- f"them from the results. Otherwise remove {overrides}."
+ f"them from the results. Otherwise remove {overrides}. Users who match {them} can then switch variants. "
+ f"{set_after_launch.remedy_after_removal()}"
)
if grade is not None:
fact += grade.sentence(them)
@@ -356,11 +269,7 @@ def _some_pinned_detail(
elif unweighted_keys:
names = " or ".join(f'"{key}"' for key in unweighted_keys)
fact += f" Nobody gets {names} at random, so their results show only these users."
- return f"{fact} {why} {fix}"
-
-
-def _percent(share: float) -> str:
- return f"{share * 100:.1f}".removesuffix(".0") + "%"
+ return f"{fact}{set_after_launch.sentence} {why} {fix}"
def _finding(
diff --git a/products/experiments/backend/health/condition_changes.py b/products/experiments/backend/health/condition_changes.py
new file mode 100644
index 000000000000..2ef19ee56e2d
--- /dev/null
+++ b/products/experiments/backend/health/condition_changes.py
@@ -0,0 +1,528 @@
+"""
+Changes of the flag's release conditions, read from the flag's activity log. Pure functions, no I/O.
+
+A condition whose criteria change, or a condition that is added or removed, moves people into or out of the flag,
+never to another variant (`get_match` in rust/feature-flags/src/flags/flag_matching.rs). A variant override does
+move people: everyone a condition with an override matches gets that variant instead of the one from the hash.
+The flag service evaluates the conditions in order, so the order moves people who match two conditions that serve
+different variants. A rollout change alone moves people in or out too, but an increase is harmless, so it is not a
+change here.
+
+Each change pairs the conditions before it with the conditions after it, as the flag page's history does, so an
+override stays with its condition when the filters of that condition change.
+
+An override set after launch that is still in place is the forced-variant check's to report, with the time it was
+set. An override that is gone leaves no state to report, so the release-condition check reports it.
+"""
+
+import json
+import itertools
+from collections import deque
+from collections.abc import Callable, Collection, Hashable, Iterator, Mapping, Sequence
+from dataclasses import replace
+from datetime import datetime
+from enum import StrEnum
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.health.context import ChangeWindow, ConditionsChange, ReleaseCondition, parse_variants
+from products.experiments.backend.health.split_changes import EXPOSURE_FREEZE_JOB_TYPES, split_variant_keys
+from products.feature_flags.backend.facade.filters import condition_aggregation, pinned_variant
+
+ReleaseConditions = tuple[ReleaseCondition, ...]
+_MAX_LOOK_ALIKES = 100
+_FOLLOWS_FLAG_AGGREGATION = "flag"
+
+
+class ConditionsChangeShape(StrEnum):
+ # pinned: the finding's subcodes. The page sends them as `finding_variant` on the health finding events.
+ VARIANT_OVERRIDE_CHANGED = "variant_override_changed"
+ CONDITIONS_CHANGED = "conditions_changed"
+ REVERTED = "reverted"
+
+
+@frozen
+class AddedOverride:
+ variant: str
+ # The number of its condition in the flag now, from 1, as the page and the flag service number them.
+ condition_number: int
+ set_at: datetime
+
+
+@frozen
+class RemovedOverride:
+ variant: str
+ # The number of its condition in the flag now. None when the condition is gone too.
+ condition_number: int | None
+ # Whether its condition matches now what it matched while the override was in place, at the same rollout, so
+ # the users it serves now are the users the override served.
+ same_audience: bool
+ # None when the override was in place before the first change.
+ set_at: datetime | None
+ # The first change of the override when it was set back, otherwise the change that removed it.
+ removed_at: datetime
+ restored_at: datetime | None
+ # From the first change of the override to the last, when its condition serves the same variant before and
+ # after them: no override, or the same one set back. Only people exposed in this window can have switched
+ # variants. None when an override was in place before the first change and is gone now.
+ temporary_window: ChangeWindow | None
+
+
+@frozen
+class EditedOverride:
+ variant: str
+ condition_number: int
+
+
+@frozen
+class OverrideChanges:
+ # Overrides set by the changes that are still in place. A changed override counts as set.
+ added: tuple[AddedOverride, ...]
+ # Overrides that the changes removed, set and removed again, or changed and set back.
+ removed: tuple[RemovedOverride, ...]
+
+
+@frozen
+class ConditionsNetEffect:
+ shape: ConditionsChangeShape
+ from_conditions: ReleaseConditions
+ to_conditions: ReleaseConditions
+ first: ConditionsChange
+ last: ConditionsChange
+ change_count: int
+ overrides: OverrideChanges
+ # Whether the conditions that no override change touches match other people after the last change.
+ audience_changed: bool
+ reordered: bool
+ # The conditions that kept their override while their criteria changed. The people who match them differently
+ # move between the override's variant and a random one.
+ edited_overrides: tuple[EditedOverride, ...]
+
+
+def conditions_change_from_activity(
+ *,
+ changed_at: datetime,
+ job_type: object,
+ before_groups: object,
+ after_groups: object,
+ before_aggregation: object,
+ after_aggregation: object,
+ before_variants: object,
+ after_variants: object,
+) -> ConditionsChange | None:
+ """The release-condition change in one activity log row of the flag, or None when the row does not change who
+ the conditions match or which variant they serve. The aggregations and variants are the flag-level ones of the
+ row."""
+ if job_type in EXPOSURE_FREEZE_JOB_TYPES:
+ return None
+ if before_groups == after_groups and before_aggregation == after_aggregation:
+ return None
+ before = parse_release_conditions(
+ before_groups,
+ flag_aggregation=before_aggregation,
+ variant_keys=split_variant_keys(parse_variants(before_variants)),
+ )
+ after = parse_release_conditions(
+ after_groups,
+ flag_aggregation=after_aggregation,
+ variant_keys=split_variant_keys(parse_variants(after_variants)),
+ )
+ if conditions_shape(before, after) is None:
+ return None
+ return ConditionsChange(changed_at=changed_at, before=before, after=after)
+
+
+def parse_release_conditions(
+ groups: object, *, flag_aggregation: object = None, variant_keys: Collection[str]
+) -> ReleaseConditions:
+ """`variant_keys` are the variants of the flag in the same version, because the flag service ignores an
+ override that names no variant of the flag."""
+ aggregation = _group_type_index(flag_aggregation)
+ return tuple(_condition(group, aggregation, variant_keys) for group in groups) if isinstance(groups, list) else ()
+
+
+def conditions_shape(before: ReleaseConditions, after: ReleaseConditions) -> ConditionsChangeShape | None:
+ """What changed between two sets of release conditions, or None when they match the same people and serve
+ them the same variants. The first match wins."""
+ return _shape(before, after, _pair_conditions(before, after))
+
+
+def _pair_conditions(before: ReleaseConditions, after: ReleaseConditions) -> dict[int, int]:
+ """
+ The condition that each condition after a change was before it, as indexes from `after` to `before`. A
+ condition without a pair was added or removed. The pairing is the one of the flag page's history
+ (`diffReleaseConditionSets` in releaseConditionsDiff.ts): an unchanged condition first, then one with the same
+ criteria, then a look-alike at the same position, then the only look-alike left.
+ """
+ pairs: dict[int, int] = {}
+ paired: set[int] = set()
+
+ def pair(after_index: int, before_index: int) -> None:
+ pairs[after_index] = before_index
+ paired.add(before_index)
+
+ def pair_first_with_same(key: Callable[[ReleaseCondition], Hashable]) -> None:
+ waiting: dict[Hashable, deque[int]] = {}
+ for index, old in enumerate(before):
+ if index not in paired:
+ waiting.setdefault(key(old), deque()).append(index)
+ for after_index, condition in enumerate(after):
+ candidates = waiting.get(key(condition))
+ if after_index not in pairs and candidates:
+ pair(after_index, candidates.popleft())
+
+ pair_first_with_same(lambda condition: condition)
+ pair_first_with_same(lambda condition: condition.criteria)
+ unpaired_after = [index for index in range(len(after)) if index not in pairs]
+ unpaired_before = [index for index in range(len(before)) if index not in paired]
+ # Look-alikes are compared each with each. A save that edits more conditions than this pairs no more of them,
+ # so they count as removed and added, which bounds the work for a flag with very many conditions.
+ if len(unpaired_after) > _MAX_LOOK_ALIKES or len(unpaired_before) > _MAX_LOOK_ALIKES:
+ return pairs
+ for after_index in unpaired_after:
+ if (
+ after_index < len(before)
+ and after_index not in paired
+ and _looks_alike(before[after_index], after[after_index])
+ ):
+ pair(after_index, after_index)
+ # A condition that was edited and moved in one save has no partner at its position. It pairs with the one
+ # condition left that it looks like. With several, no pair is safe, so they count as removed and added.
+ for after_index in unpaired_after:
+ if after_index in pairs:
+ continue
+ candidates = [
+ index
+ for index in unpaired_before
+ if index not in paired and _looks_alike(before[index], after[after_index])
+ ]
+ if len(candidates) == 1:
+ pair(after_index, candidates[0])
+ return pairs
+
+
+def conditions_net_effect(
+ changes: Sequence[ConditionsChange], *, current: ReleaseConditions | None
+) -> ConditionsNetEffect | None:
+ """
+ `current` are the flag's conditions now, which the condition numbers follow. A save that changes nothing this
+ module reads, such as the order of conditions without an override, is no change, so it can come after the
+ last change and renumber the conditions. None numbers them as after the last change.
+ """
+ if not changes:
+ return None
+ first = changes[0]
+ last = changes[-1]
+ followed = _rejoin(_follow(changes), first.before, last.after)
+ now = last.after if current is None else current
+ now_ids = (
+ followed.end_ids
+ if current is None
+ else _carry_ids(last.after, followed.end_ids, current, itertools.count(_next_id(followed)))
+ )
+ numbers = {condition_id: number for number, condition_id in enumerate(now_ids, start=1)}
+ audience_now = {condition_id: _audience(condition) for condition_id, condition in zip(now_ids, now)}
+ overrides, touched = _override_changes(followed, first.before, last.after, numbers, audience_now)
+ start_index = {condition_id: index for index, condition_id in enumerate(followed.start_ids)}
+ net_pairs = {
+ index: start_index[condition_id]
+ for index, condition_id in enumerate(followed.end_ids)
+ if condition_id in start_index
+ }
+ return ConditionsNetEffect(
+ shape=_shape(first.before, last.after, net_pairs) or ConditionsChangeShape.REVERTED,
+ from_conditions=first.before,
+ to_conditions=last.after,
+ first=first,
+ last=last,
+ change_count=len(changes),
+ overrides=overrides,
+ audience_changed=_criteria_without(first.before, followed.start_ids, touched)
+ != _criteria_without(last.after, followed.end_ids, touched),
+ reordered=_reordered(first.before, last.after, net_pairs),
+ edited_overrides=tuple(
+ EditedOverride(variant=variant, condition_number=numbers[condition_id])
+ for end_index, start_index in sorted(net_pairs.items())
+ for condition_id, old, new in [
+ (followed.end_ids[end_index], first.before[start_index], last.after[end_index])
+ ]
+ for variant in [new.variant]
+ if variant is not None
+ and condition_id not in touched
+ and condition_id in numbers
+ and new.rollout > 0
+ and new.criteria != old.criteria
+ ),
+ )
+
+
+@frozen
+class _VariantChange:
+ condition_id: int
+ changed_at: datetime
+ before: str | None
+ after: str | None
+ before_audience: tuple[str, float] | None
+
+
+@frozen
+class _Followed:
+ start_ids: tuple[int, ...]
+ end_ids: tuple[int, ...]
+ variant_changes: tuple[_VariantChange, ...]
+
+
+def _follow(changes: Sequence[ConditionsChange]) -> _Followed:
+ new_ids = itertools.count()
+ start_ids = tuple(next(new_ids) for _ in changes[0].before)
+ conditions, ids = changes[0].before, start_ids
+ variant_changes = []
+ for change in changes:
+ # A save that changes only what this module ignores, such as a rollout, is not a change, and neither is an
+ # exposure freeze. A change can then start from other conditions than the ones the change before it ended
+ # with, so those are paired too.
+ before_ids = _carry_ids(conditions, ids, change.before, new_ids)
+ after_ids = _carry_ids(change.before, before_ids, change.after, new_ids)
+ before = dict(zip(before_ids, change.before))
+ after = dict(zip(after_ids, change.after))
+ for condition_id in dict.fromkeys([*before_ids, *after_ids]):
+ old, new = before.get(condition_id), after.get(condition_id)
+ if _moves_people(old, new):
+ variant_changes.append(
+ _VariantChange(
+ condition_id=condition_id,
+ changed_at=change.changed_at,
+ before=_variant(old),
+ after=_variant(new),
+ before_audience=_audience(old) if old is not None else None,
+ )
+ )
+ conditions, ids = change.after, after_ids
+ return _Followed(start_ids=start_ids, end_ids=ids, variant_changes=tuple(variant_changes))
+
+
+def _rejoin(followed: _Followed, start: ReleaseConditions, end: ReleaseConditions) -> _Followed:
+ """A condition that one change removes and a later change adds back is two conditions to the pairing of each
+ change. Pairing the conditions gone by the end with the ones new since the start joins them again."""
+ start_ids, end_ids = set(followed.start_ids), set(followed.end_ids)
+ gone = [index for index, condition_id in enumerate(followed.start_ids) if condition_id not in end_ids]
+ new = [index for index, condition_id in enumerate(followed.end_ids) if condition_id not in start_ids]
+ pairs = _pair_conditions(tuple(start[index] for index in gone), tuple(end[index] for index in new))
+ joined = {
+ followed.end_ids[new[new_index]]: followed.start_ids[gone[gone_index]]
+ for new_index, gone_index in pairs.items()
+ }
+ return _Followed(
+ start_ids=followed.start_ids,
+ end_ids=tuple(joined.get(condition_id, condition_id) for condition_id in followed.end_ids),
+ variant_changes=tuple(
+ replace(change, condition_id=joined.get(change.condition_id, change.condition_id))
+ for change in followed.variant_changes
+ ),
+ )
+
+
+def _next_id(followed: _Followed) -> int:
+ ids = [*followed.start_ids, *followed.end_ids, *(change.condition_id for change in followed.variant_changes)]
+ return max(ids, default=-1) + 1
+
+
+def _carry_ids(
+ before: ReleaseConditions, ids: tuple[int, ...], after: ReleaseConditions, new_ids: Iterator[int]
+) -> tuple[int, ...]:
+ pairs = _pair_conditions(before, after)
+ return tuple(ids[pairs[index]] if index in pairs else next(new_ids) for index in range(len(after)))
+
+
+def _override_changes(
+ followed: _Followed,
+ start: ReleaseConditions,
+ end: ReleaseConditions,
+ numbers: Mapping[int, int],
+ audience_now: Mapping[int, tuple[str, float]],
+) -> tuple[OverrideChanges, set[int]]:
+ """The override changes, and the ids of the conditions they touch."""
+ start_variants = {condition_id: condition.variant for condition_id, condition in zip(followed.start_ids, start)}
+ end_variants = {condition_id: condition.variant for condition_id, condition in zip(followed.end_ids, end)}
+ added: list[AddedOverride] = []
+ removed: list[RemovedOverride] = []
+ by_condition: dict[int, list[_VariantChange]] = {}
+ for change in followed.variant_changes:
+ by_condition.setdefault(change.condition_id, []).append(change)
+ for condition_id, changes in by_condition.items():
+ start_variant = start_variants.get(condition_id)
+ end_variant = end_variants.get(condition_id)
+ number = numbers.get(condition_id)
+ if end_variant is not None and end_variant != start_variant:
+ set_at = _last_set(changes, end_variant)
+ if set_at is not None and number is not None:
+ added.append(AddedOverride(variant=end_variant, condition_number=number, set_at=set_at))
+ elif end_variant is not None:
+ removed.append(
+ RemovedOverride(
+ variant=end_variant,
+ condition_number=number,
+ same_audience=changes[0].before_audience == audience_now.get(condition_id),
+ set_at=None,
+ removed_at=changes[0].changed_at,
+ restored_at=changes[-1].changed_at,
+ temporary_window=ChangeWindow(start=changes[0].changed_at, end=changes[-1].changed_at),
+ )
+ )
+ else:
+ removal_index = next(
+ (index for index in range(len(changes) - 1, -1, -1) if changes[index].after is None), None
+ )
+ if removal_index is None:
+ continue
+ removal = changes[removal_index]
+ if removal.before is None:
+ continue
+ removed.append(
+ RemovedOverride(
+ variant=removal.before,
+ condition_number=number,
+ same_audience=removal.before_audience == audience_now.get(condition_id),
+ set_at=None
+ if removal.before == start_variant
+ else _last_set(changes[:removal_index], removal.before),
+ removed_at=removal.changed_at,
+ restored_at=None,
+ temporary_window=None
+ if start_variant is not None
+ else ChangeWindow(start=changes[0].changed_at, end=removal.changed_at),
+ )
+ )
+ touched = set(by_condition)
+ return OverrideChanges(added=tuple(added), removed=tuple(removed)), touched
+
+
+def _last_set(changes: Sequence[_VariantChange], variant: str) -> datetime | None:
+ return next((change.changed_at for change in reversed(changes) if change.after == variant), None)
+
+
+def _shape(
+ before: ReleaseConditions, after: ReleaseConditions, pairs: Mapping[int, int]
+) -> ConditionsChangeShape | None:
+ paired = set(pairs.values())
+ sides = [
+ *((before[before_index], after[after_index]) for after_index, before_index in pairs.items()),
+ *((condition, None) for index, condition in enumerate(before) if index not in paired),
+ *((None, condition) for index, condition in enumerate(after) if index not in pairs),
+ ]
+ if any(_moves_people(old, new) for old, new in sides) or _reordered(before, after, pairs):
+ return ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED
+ # When no variant moves, the order does not decide who is in the flag.
+ if sorted(condition.criteria for condition in before) != sorted(condition.criteria for condition in after):
+ return ConditionsChangeShape.CONDITIONS_CHANGED
+ return None
+
+
+def _moves_people(before: ReleaseCondition | None, after: ReleaseCondition | None) -> bool:
+ # A condition at 0% rollout serves nobody, so its override moves nobody.
+ return _variant(before) != _variant(after) and any(
+ condition is not None and condition.rollout > 0 for condition in (before, after)
+ )
+
+
+def _reordered(before: ReleaseConditions, after: ReleaseConditions, pairs: Mapping[int, int]) -> bool:
+ # A person who matches two conditions gets the variant of the first one, the override or the hash's. Walking
+ # the conditions in their new order, a condition moved ahead of one with another variant when that one sat
+ # later before the change. Keeping the latest position and the latest of any other variant finds it in one pass.
+ latest: tuple[int, str | None] | None = None
+ latest_of_other_variant = -1
+ for after_index, before_index in sorted(pairs.items()):
+ if before[before_index].rollout <= 0 or after[after_index].rollout <= 0:
+ continue
+ variant = after[after_index].variant
+ if latest is not None:
+ latest_position, latest_variant = latest
+ if (latest_position if latest_variant != variant else latest_of_other_variant) > before_index:
+ return True
+ if variant == latest_variant:
+ latest = (max(latest_position, before_index), variant)
+ elif before_index > latest_position:
+ latest_of_other_variant = latest_position
+ latest = (before_index, variant)
+ else:
+ latest_of_other_variant = max(latest_of_other_variant, before_index)
+ else:
+ latest = (before_index, variant)
+ return False
+
+
+def _criteria_without(conditions: ReleaseConditions, ids: tuple[int, ...], touched: set[int]) -> list[str]:
+ return sorted(condition.criteria for condition, condition_id in zip(conditions, ids) if condition_id not in touched)
+
+
+def _variant(condition: ReleaseCondition | None) -> str | None:
+ return condition.variant if condition is not None else None
+
+
+def _audience(condition: ReleaseCondition) -> tuple[str, float]:
+ return condition.criteria, condition.rollout
+
+
+def _looks_alike(old: ReleaseCondition, new: ReleaseCondition) -> bool:
+ # Two conditions look alike when they match the same kind of thing: the same group type, and a shared property
+ # key or no properties on either. Otherwise an unrelated new condition would read as an edit.
+ if old.aggregation != new.aggregation:
+ return False
+ if not old.property_keys and not new.property_keys:
+ return True
+ return bool(old.property_keys & new.property_keys)
+
+
+def _condition(group: object, flag_aggregation: int | None, variant_keys: Collection[str]) -> ReleaseCondition:
+ if not isinstance(group, dict):
+ return ReleaseCondition(
+ criteria=json.dumps(group, sort_keys=True, default=str),
+ aggregation=None,
+ property_keys=frozenset(),
+ rollout=100,
+ variant=None,
+ )
+ properties = group.get("properties")
+ rollout = group.get("rollout_percentage")
+ aggregation = condition_aggregation(group, flag_aggregation)
+ return ReleaseCondition(
+ criteria=json.dumps(
+ [
+ # A change of the flag-level aggregation is the bucketing check's to report, so a condition that
+ # follows it keeps its criteria.
+ _FOLLOWS_FLAG_AGGREGATION if aggregation == flag_aggregation else aggregation,
+ # The filters of a condition all have to match, so their order does not matter.
+ sorted(_property_signature(prop) for prop in properties) if isinstance(properties, list) else None,
+ ],
+ sort_keys=True,
+ default=str,
+ ),
+ aggregation=_group_type_index(aggregation),
+ property_keys=frozenset(_property_key(prop) for prop in properties)
+ if isinstance(properties, list)
+ else frozenset(),
+ # The flag service reads a missing rollout as 100%.
+ rollout=rollout if isinstance(rollout, int | float) and not isinstance(rollout, bool) else 100,
+ variant=pinned_variant(group, variant_keys),
+ )
+
+
+def _property_signature(prop: object) -> str:
+ # The fields that decide a match, as the flag page's history compares them (releaseConditionsDiff.ts).
+ if not isinstance(prop, dict):
+ return json.dumps(prop, sort_keys=True, default=str)
+ return json.dumps(
+ [prop.get(field) for field in ("type", "key", "operator", "value", "group_type_index")],
+ sort_keys=True,
+ default=str,
+ )
+
+
+def _property_key(prop: object) -> str:
+ if not isinstance(prop, dict):
+ return json.dumps(prop, sort_keys=True, default=str)
+ return f"{prop.get('type') or ''}:{prop.get('key') or ''}"
+
+
+def _group_type_index(value: object) -> int | None:
+ return value if isinstance(value, int) and not isinstance(value, bool) else None
diff --git a/products/experiments/backend/health/condition_set_traffic.py b/products/experiments/backend/health/condition_set_traffic.py
index 193c160476e1..a9c5384ecec2 100644
--- a/products/experiments/backend/health/condition_set_traffic.py
+++ b/products/experiments/backend/health/condition_set_traffic.py
@@ -37,13 +37,16 @@ from posthog.exceptions_capture import capture_exception
from posthog.utils import safe_cache_add
from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess
-from products.experiments.backend.health.checks.forced_variant import (
- TRAFFIC_WINDOW,
- has_too_few_people,
- needs_condition_set_traffic,
-)
+from products.experiments.backend.health.checks.conditions_changed import removed_overrides_to_grade
+from products.experiments.backend.health.checks.forced_variant import overrides_to_grade
from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext
from products.experiments.backend.health.exposure_gate import UNKNOWN, cache_failed_scan
+from products.experiments.backend.health.pinned_share import (
+ TRAFFIC_WINDOW,
+ PinnedConditions,
+ PinnedShareUnknownReason,
+ grade_pinned_share,
+)
from products.experiments.backend.hogql_queries.exposure_query_logic import (
DEFAULT_EXPOSURE_EVENT,
exposure_event_name_filter,
@@ -91,6 +94,24 @@ def load_condition_set_traffic(
return None
+def needs_condition_set_traffic(ctx: HealthContext) -> bool:
+ """Whether a check grades its finding by the users that pinned conditions served: the forced-variant check for
+ the overrides in place, and the release-condition check for an override removed after launch."""
+ return ctx.is_launched and any(pins.weighted for pins in _pins_to_grade(ctx))
+
+
+def has_too_few_people(ctx: HealthContext, traffic: ConditionSetTraffic) -> bool:
+ """Whether a check leaves the share of `traffic` unknown only because too few people count."""
+ return any(
+ pins.weighted and grade_pinned_share(pins, traffic).unknown_reason == PinnedShareUnknownReason.TOO_FEW_PEOPLE
+ for pins in _pins_to_grade(ctx)
+ )
+
+
+def _pins_to_grade(ctx: HealthContext) -> list[PinnedConditions]:
+ return [pins for pins in (overrides_to_grade(ctx), removed_overrides_to_grade(ctx)) if pins is not None]
+
+
def denied_scanned_properties(reader_access: ExperimentHealthReaderAccess) -> frozenset[str]:
"""The properties that the scan reads and that the reader is denied. The scan does not run when there are any."""
return reader_access.denied_event_properties & _SCANNED_PROPERTIES
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index ff10562b394b..6157ce49ef45 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -46,16 +46,42 @@ class BucketingChange:
after: str | int | bool | None
+@frozen
+class ReleaseCondition:
+ # The group type and property filters of the condition, as a signature that is equal for two conditions
+ # that match the same people under the flag-level aggregation of their save. The rollout is not part of it.
+ criteria: str
+ # The group type index that the condition matches, None for persons, and the keys of its property filters.
+ # The flag page's history pairs two conditions that share them as one condition edited.
+ aggregation: int | None
+ property_keys: frozenset[str]
+ rollout: float
+ variant: str | None
+
+
+@frozen
+class ConditionsChange:
+ changed_at: datetime
+ before: tuple[ReleaseCondition, ...]
+ after: tuple[ReleaseCondition, ...]
+
+
# A change of the flag after launch that can move exposed people. A new kind needs its class here, an
# `ExperimentHealthFlagChangeKind`, its read and its set-back question in `exposure_gate.py`, its row in
# `inputs_payload.py`, and its check.
-FlagChange = SplitChange | BucketingChange
+FlagChange = SplitChange | BucketingChange | ConditionsChange
def changes_of_kind[C: FlagChange](changes: tuple[FlagChange, ...], kind: type[C]) -> tuple[C, ...]:
return tuple(change for change in changes if isinstance(change, kind))
+@frozen
+class ChangeWindow:
+ start: datetime
+ end: datetime
+
+
@frozen
class FlagHistory:
"""The changes of the flag after launch that can have moved exposed people, and whether anyone was exposed
@@ -73,6 +99,12 @@ class FlagHistory:
# Whether anyone was exposed between the first and the last counted change of a kind. Asked only when the last
# sets back what the first changed, so a kind that was not asked is absent. None when unknown.
exposed_between: Mapping[ExperimentHealthFlagChangeKind, bool | None]
+ # The flag's release conditions now. A save that no kind counts can renumber them after the last change, and
+ # the findings and the traffic number the conditions as they are now.
+ release_conditions: tuple[ReleaseCondition, ...]
+ # Whether anyone was exposed while an override was set and removed again, or removed and set back. A user
+ # exposed only outside that window got the same variant throughout. None when unknown.
+ exposed_during_overrides: Mapping[ChangeWindow, bool | None]
def of_kind[C: FlagChange](self, kind: type[C]) -> tuple[C, ...]:
return changes_of_kind(self.changes, kind)
@@ -89,6 +121,14 @@ class FlagHistoryNotCheckedReason(StrEnum):
EXPOSURE_FILTERS = "exposure_filters"
EXPOSURE_ACTION = "exposure_action"
EXPOSURE_CRITERIA = "exposure_criteria"
+ # The history is not read past these bounds, because the work grows with the saves and the conditions, and the
+ # experiment page asks on every load.
+ TOO_MANY_SAVES = "too_many_saves"
+ TOO_MANY_CONDITIONS = "too_many_conditions"
+
+
+MAX_FLAG_SAVES = 100
+MAX_RELEASE_CONDITIONS = 100
@frozen
diff --git a/products/experiments/backend/health/exposure_gate.py b/products/experiments/backend/health/exposure_gate.py
index 97b3a2960387..2b62f12ae581 100644
--- a/products/experiments/backend/health/exposure_gate.py
+++ b/products/experiments/backend/health/exposure_gate.py
@@ -1,6 +1,7 @@
"""
-Loads the changes of a running experiment's flag since launch that can move exposed people: the split and the
-settings that decide the variant draw. Then asks ClickHouse whether anyone was exposed before each change. A change before the first exposure moved nobody, so it does not count.
+Loads the changes of a running experiment's flag since launch that can move exposed people: the split, the
+settings that decide the variant draw, and the release conditions. Then asks ClickHouse whether anyone was exposed
+before each change. A change before the first exposure moved nobody, so it does not count.
The answer for a past window does not change, so it is cached per window and shared by every reader. A slow
answer is then paid once per change, not on every load of the experiment page. A failed query is cached as
@@ -18,6 +19,7 @@ from collections.abc import Sequence
from dataclasses import replace
from datetime import datetime, timedelta
from time import monotonic
+from typing import Any
from django.core.cache import cache
from django.db.models import F, Func, JSONField, Q, Value
@@ -52,9 +54,19 @@ from products.experiments.backend.health.bucketing_changes import (
changes_that_apply,
conditions_follow_flag_aggregation,
)
+from products.experiments.backend.health.condition_changes import (
+ ConditionsChangeShape,
+ conditions_change_from_activity,
+ conditions_net_effect,
+ parse_release_conditions,
+)
from products.experiments.backend.health.context import (
+ MAX_FLAG_SAVES,
+ MAX_RELEASE_CONDITIONS,
BucketingChange,
BucketingSetting,
+ ChangeWindow,
+ ConditionsChange,
FlagChange,
FlagHistory,
FlagHistoryNotChecked,
@@ -78,6 +90,9 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
from products.experiments.backend.models.experiment import Experiment
MAX_EXPOSURE_QUERIES = 20
+# Each override window costs a cache read, and a query when the answer is not cached. A window past this stays
+# unknown, so its override stays reported.
+MAX_OVERRIDE_WINDOWS = 20
EXPOSURE_QUERY_TIMEOUT_SECONDS = 10
# No query starts after this, so one request takes at most this plus one query's timeout.
REQUEST_BUDGET_SECONDS = 10
@@ -237,6 +252,8 @@ class _ExposureGate:
return None
settled = answer or timezone.now() - window.end > _SETTLED_AFTER
cache.set(cache_key, answer, _ANSWER_CACHE_SECONDS if settled else _RECENT_NOBODY_CACHE_SECONDS)
+ # Two kinds can ask the same window, and a prefetched miss would otherwise send the second to ClickHouse.
+ self._prefetched[cache_key] = answer
return answer
def _cache_key(self, window: _Window) -> str:
@@ -283,7 +300,11 @@ def _load_flag_history(
changes=(),
)
- read = _flag_changes_since(team_id=experiment.team_id, feature_flag_id=feature_flag_id, since=launched_at)
+ rows = _flag_rows_since(team_id=experiment.team_id, feature_flag_id=feature_flag_id, since=launched_at)
+ too_large = _too_large(rows)
+ if too_large is not None:
+ return FlagHistoryNotChecked(reason=too_large, launched_at=launched_at, team_timezone=team_timezone, changes=())
+ read = _flag_changes(rows, feature_flag_id=feature_flag_id)
flag = experiment.feature_flag
trust_flag_aggregation = conditions_follow_flag_aggregation(flag.aggregation_group_type_index, flag.conditions)
bucketing = changes_of_kind(read, BucketingChange)
@@ -316,6 +337,12 @@ def _load_flag_history(
counted_since=None,
exposed_before_first=False,
exposed_between={},
+ release_conditions=parse_release_conditions(
+ flag.conditions,
+ flag_aggregation=flag.aggregation_group_type_index,
+ variant_keys=split_variant_keys(parse_variants(flag.variants)),
+ ),
+ exposed_during_overrides={},
)
times = _change_times(changes)
if not times:
@@ -341,7 +368,11 @@ def _load_flag_history(
break
if history.counted_since is None:
return history
- return replace(history, exposed_between=_exposed_between_set_backs(gate, timeline, history))
+ return replace(
+ history,
+ exposed_between=_exposed_between_set_backs(gate, timeline, history),
+ exposed_during_overrides=_exposed_during_overrides(gate, timeline, history),
+ )
def _not_checked_reason(
@@ -440,20 +471,38 @@ def _exposed_between_set_backs(
exposed_between[ExperimentHealthFlagChangeKind.BUCKETING] = _exposed_between(
gate, timeline, start=bucketing.first.changed_at, end=bucketing.last.changed_at
)
+ conditions = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
+ if conditions and conditions.shape == ConditionsChangeShape.REVERTED:
+ exposed_between[ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS] = _exposed_between(
+ gate, timeline, start=conditions.first.changed_at, end=conditions.last.changed_at
+ )
return exposed_between
+def _exposed_during_overrides(
+ gate: _ExposureGate, timeline: _Timeline, history: FlagHistory
+) -> dict[ChangeWindow, bool | None]:
+ """Asks, per override that was set and removed again or removed and set back, whether anyone was exposed in
+ between. The set-back question above covers only the span of every change of a kind."""
+ effect = conditions_net_effect(history.counted(ConditionsChange), current=history.release_conditions)
+ windows = [] if effect is None else [override.temporary_window for override in effect.overrides.removed]
+ asked = [window for window in dict.fromkeys(windows) if window is not None][:MAX_OVERRIDE_WINDOWS]
+ return {window: _exposed_between(gate, timeline, start=window.start, end=window.end) for window in asked}
+
+
_PROJECTED = (
"before_variants",
"after_variants",
+ "before_groups",
+ "after_groups",
"before_aggregation",
"after_aggregation",
*(f"{side}_{setting}" for side in ("before", "after") for setting in _SETTING_FIELDS),
)
-def _flag_changes_since(*, team_id: int, feature_flag_id: int, since: datetime) -> tuple[FlagChange, ...]:
- rows = (
+def _flag_rows_since(*, team_id: int, feature_flag_id: int, since: datetime) -> list[dict[str, Any]]:
+ return list(
ActivityLog.objects.filter(
_changes_any_field(_FILTERS, *_SETTING_FIELDS.values()),
team_id=team_id,
@@ -467,13 +516,28 @@ def _flag_changes_since(*, team_id: int, feature_flag_id: int, since: datetime)
job_type=KT("detail__trigger__job_type"),
before_variants=_changed(_FILTERS, "before.multivariate.variants"),
after_variants=_changed(_FILTERS, "after.multivariate.variants"),
+ before_groups=_changed(_FILTERS, "before.groups"),
+ after_groups=_changed(_FILTERS, "after.groups"),
before_aggregation=_changed(_FILTERS, "before.aggregation_group_type_index"),
after_aggregation=_changed(_FILTERS, "after.aggregation_group_type_index"),
**{f"before_{setting}": _changed(field, "before") for setting, field in _SETTING_FIELDS.items()},
**{f"after_{setting}": _changed(field, "after") for setting, field in _SETTING_FIELDS.items()},
)
- .values("created_at", "job_type", *_PROJECTED)
+ .values("created_at", "job_type", *_PROJECTED)[: MAX_FLAG_SAVES + 1]
)
+
+
+def _too_large(rows: list[dict[str, Any]]) -> FlagHistoryNotCheckedReason | None:
+ if len(rows) > MAX_FLAG_SAVES:
+ return FlagHistoryNotCheckedReason.TOO_MANY_SAVES
+ for row in rows:
+ for groups in (row["before_groups"], row["after_groups"]):
+ if isinstance(groups, list) and len(groups) > MAX_RELEASE_CONDITIONS:
+ return FlagHistoryNotCheckedReason.TOO_MANY_CONDITIONS
+ return None
+
+
+def _flag_changes(rows: list[dict[str, Any]], *, feature_flag_id: int) -> tuple[FlagChange, ...]:
changes: list[FlagChange] = []
for row in rows:
changed_at = row["created_at"]
@@ -493,6 +557,18 @@ def _flag_changes_since(*, team_id: int, feature_flag_id: int, since: datetime)
after=_settings(row, "after"),
)
)
+ conditions_change = conditions_change_from_activity(
+ changed_at=changed_at,
+ job_type=row["job_type"],
+ before_groups=row["before_groups"],
+ after_groups=row["after_groups"],
+ before_aggregation=row["before_aggregation"],
+ after_aggregation=row["after_aggregation"],
+ before_variants=row["before_variants"],
+ after_variants=row["after_variants"],
+ )
+ if conditions_change is not None:
+ changes.append(conditions_change)
return tuple(changes)
diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.py
index f534beabf8be..e10e79f85ea1 100644
--- a/products/experiments/backend/health/inputs_payload.py
+++ b/products/experiments/backend/health/inputs_payload.py
@@ -7,16 +7,20 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFlagChangeKind,
ExperimentHealthInputsTrafficStatus,
)
-from products.experiments.backend.health.checks.forced_variant import needs_condition_set_traffic
-from products.experiments.backend.health.condition_set_traffic import denied_scanned_properties
+from products.experiments.backend.health.condition_set_traffic import (
+ denied_scanned_properties,
+ needs_condition_set_traffic,
+)
from products.experiments.backend.health.context import (
BucketingChange,
BucketingSetting,
+ ConditionsChange,
ConditionSetTraffic,
FlagChange,
FlagHistory,
FlagHistoryNotChecked,
FlagState,
+ ReleaseCondition,
Split,
SplitChange,
)
@@ -78,6 +82,16 @@ def _health_inputs_bucketing_value(value: str | int | bool | None) -> str:
return value if isinstance(value, str) else json.dumps(value)
+def _health_inputs_conditions_text(conditions: tuple[ReleaseCondition, ...]) -> str:
+ count = f"{len(conditions)} {'condition' if len(conditions) == 1 else 'conditions'}"
+ overrides = [
+ f"set {number} overrides to {condition.variant}"
+ for number, condition in enumerate(conditions, start=1)
+ if condition.variant
+ ]
+ return ", ".join([count, *overrides])
+
+
def _health_inputs_flag_change(change: FlagChange) -> dict[str, Any]:
"""A change as the staff view lists it, with its values as text, so the view needs no code per kind."""
match change:
@@ -95,6 +109,13 @@ def _health_inputs_flag_change(change: FlagChange) -> dict[str, Any]:
"before": _health_inputs_bucketing_value(change.before),
"after": _health_inputs_bucketing_value(change.after),
}
+ case ConditionsChange():
+ return {
+ "kind": ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS,
+ "changed": "Release conditions",
+ "before": _health_inputs_conditions_text(change.before),
+ "after": _health_inputs_conditions_text(change.after),
+ }
def _health_inputs_flag_history(history: FlagHistory | FlagHistoryNotChecked) -> dict[str, Any]:
diff --git a/products/experiments/backend/health/pinned_share.py b/products/experiments/backend/health/pinned_share.py
new file mode 100644
index 000000000000..7ced02c0bb7a
--- /dev/null
+++ b/products/experiments/backend/health/pinned_share.py
@@ -0,0 +1,199 @@
+"""
+The share of a variant's users that release conditions with a variant override serve, which grades the findings
+about overrides: the forced-variant check grades the overrides in place, and the release-condition check the ones
+removed after launch. Pure functions, no I/O: `health/condition_set_traffic.py` loads the traffic.
+"""
+
+from collections.abc import Sequence
+from datetime import datetime, timedelta
+from enum import StrEnum
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
+from products.experiments.backend.health.context import ConditionSetTraffic, ReachableCondition, Split
+from products.experiments.backend.health.split_changes import format_change_time
+
+# A finding is graded by the largest share that pinned users make up of one variant's users, in this window, or
+# since launch or the flag's last save if later. The bias is per variant, so a few pinned users weigh more in a
+# small variant. Below the first share they can't move a variant's results much, so there is no finding. Above the
+# second they are a warning.
+TRAFFIC_WINDOW = timedelta(days=7)
+_NO_FINDING_BELOW_SHARE = 0.01
+_WARNING_ABOVE_SHARE = 0.05
+# With fewer people, the share of a few pinned people is mostly chance.
+_MIN_MATCHED_PEOPLE = 100
+# Only calls that the flag service evaluated name a condition set. When fewer calls than this share did, the
+# pinned users can be among the calls that name none, so a share under the first threshold does not clear the
+# finding. A share above it still counts, because the pinned users it found are real.
+_MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR = 0.5
+
+
+class PinnedShareUnknownReason(StrEnum):
+ # pinned: the finding's `evidence.reason`, which the health finding events track.
+ NOT_MEASURED = "not_measured"
+ TOO_FEW_PEOPLE = "too_few_people"
+ # Most calls carry no version: local evaluation in a server SDK, a bootstrap, or the customer's own assignment.
+ EVALUATED_OUTSIDE_FLAG_SERVICE = "evaluated_outside_flag_service"
+
+
+@frozen
+class PinnedConditions:
+ """The release conditions with a variant override that a finding is graded by."""
+
+ # The conditions pinned to a variant with a weight in the split, in evaluation order.
+ weighted: tuple[ReachableCondition, ...]
+ # The conditions pinned to a variant with 0% in the split. The split sends nobody there, so they don't bias the
+ # other variants, but that variant's results show only the pinned users.
+ unweighted: tuple[ReachableCondition, ...]
+ split: Split
+ # False when some pinned users have no condition left that counts them, so a small share can't clear them.
+ all_counted: bool
+
+
+def pinned_conditions(
+ conditions: Sequence[ReachableCondition], split: Split, *, all_counted: bool = True
+) -> PinnedConditions:
+ unweighted_keys = {variant.key for variant in split if variant.rollout_percentage == 0}
+ return PinnedConditions(
+ weighted=tuple(condition for condition in conditions if condition.pinned_variant not in unweighted_keys),
+ unweighted=tuple(condition for condition in conditions if condition.pinned_variant in unweighted_keys),
+ split=split,
+ all_counted=all_counted,
+ )
+
+
+@frozen
+class PinnedShareMeasurement:
+ most_pinned_variant: str
+ pinned_people: int
+ # The variant's pinned people, and its weight in the split of the people assigned at random.
+ variant_people: float
+ # The people that a condition set served, without those pinned to a variant with 0% in the split.
+ matched_people: int
+ remotely_evaluated_share: float | None
+ since: datetime
+ team_timezone: str
+
+ @property
+ def share(self) -> float:
+ return self.pinned_people / self.variant_people if self.variant_people else 0.0
+
+
+@frozen
+class PinnedShareGrade:
+ """The share that sets the severity of a finding about pinned conditions, or the reason it is unknown."""
+
+ measurement: PinnedShareMeasurement | None
+ share: float | None
+ unknown_reason: PinnedShareUnknownReason | None
+
+ def evidence(self) -> dict[str, str | int | float | None]:
+ evidence: dict[str, str | int | float | None] = {}
+ if self.measurement is not None:
+ remotely_evaluated_share = self.measurement.remotely_evaluated_share
+ evidence["most_pinned_variant"] = self.measurement.most_pinned_variant
+ evidence["pinned_people"] = self.measurement.pinned_people
+ evidence["variant_people"] = round(self.measurement.variant_people)
+ evidence["matched_people"] = self.measurement.matched_people
+ evidence["remotely_evaluated_share"] = (
+ None if remotely_evaluated_share is None else round(remotely_evaluated_share, 4)
+ )
+ if self.share is not None:
+ evidence["pinned_share"] = round(self.share, 4)
+ if self.unknown_reason is not None:
+ evidence["reason"] = self.unknown_reason.value
+ return evidence
+
+ def sentence(self, them: str) -> str:
+ """The measured share in words, or why it can't be measured, after a leading space. Empty otherwise."""
+ measurement = self.measurement
+ if self.share is not None and measurement is not None:
+ return (
+ f" Since {format_change_time(measurement.since, measurement.team_timezone)}, about "
+ f'{percent(self.share)} of the users who got "{measurement.most_pinned_variant}" matched {them} '
+ f"({measurement.pinned_people:,} of about {round(measurement.variant_people):,})."
+ )
+ if self.unknown_reason == PinnedShareUnknownReason.EVALUATED_OUTSIDE_FLAG_SERVICE:
+ return (
+ " Most of this flag's calls are evaluated locally in a server SDK, bootstrapped, or assigned in your "
+ f"own code, so PostHog can't measure how many users match {them}."
+ )
+ return ""
+
+
+def grade_pinned_share(pins: PinnedConditions, traffic: ConditionSetTraffic | None) -> PinnedShareGrade:
+ if traffic is None:
+ return PinnedShareGrade(measurement=None, share=None, unknown_reason=PinnedShareUnknownReason.NOT_MEASURED)
+ measurement = _measure(pins, traffic)
+ remotely_evaluated_share = measurement.remotely_evaluated_share
+ unknown_reason = (
+ PinnedShareUnknownReason.EVALUATED_OUTSIDE_FLAG_SERVICE
+ if remotely_evaluated_share is not None and remotely_evaluated_share < _MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR
+ else PinnedShareUnknownReason.TOO_FEW_PEOPLE
+ )
+ if measurement.matched_people < _MIN_MATCHED_PEOPLE:
+ return PinnedShareGrade(measurement=measurement, share=None, unknown_reason=unknown_reason)
+ share = measurement.share
+ clearing_needs_more_calls = (
+ remotely_evaluated_share is None or remotely_evaluated_share < _MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR
+ )
+ if share < _NO_FINDING_BELOW_SHARE and clearing_needs_more_calls:
+ return PinnedShareGrade(measurement=measurement, share=None, unknown_reason=unknown_reason)
+ if share < _NO_FINDING_BELOW_SHARE and not pins.all_counted:
+ return PinnedShareGrade(
+ measurement=measurement, share=None, unknown_reason=PinnedShareUnknownReason.NOT_MEASURED
+ )
+ return PinnedShareGrade(measurement=measurement, share=share, unknown_reason=None)
+
+
+def pinned_share_severity(grade: PinnedShareGrade | None) -> ExperimentHealthFindingSeverity | None:
+ """The severity of a finding graded by a pinned share. None when the share is too small for a finding."""
+ if grade is None or grade.share is None:
+ return ExperimentHealthFindingSeverity.INFO
+ if grade.share < _NO_FINDING_BELOW_SHARE:
+ return None
+ return (
+ ExperimentHealthFindingSeverity.WARNING
+ if grade.share > _WARNING_ABOVE_SHARE
+ else ExperimentHealthFindingSeverity.INFO
+ )
+
+
+def pinned_variant_keys(conditions: Sequence[ReachableCondition]) -> list[str]:
+ return list(dict.fromkeys(condition.pinned_variant for condition in conditions if condition.pinned_variant))
+
+
+def percent(share: float) -> str:
+ return f"{share * 100:.1f}".removesuffix(".0") + "%"
+
+
+def _measure(pins: PinnedConditions, traffic: ConditionSetTraffic) -> PinnedShareMeasurement:
+ excluded_numbers = {condition.number for condition in pins.unweighted}
+ matched_people = sum(people for number, people in traffic.people_by_set.items() if number not in excluded_numbers)
+ pinned_by_variant = dict.fromkeys(pinned_variant_keys(pins.weighted), 0)
+ for condition in pins.weighted:
+ if condition.pinned_variant is not None:
+ pinned_by_variant[condition.pinned_variant] += traffic.people_by_set.get(condition.number, 0)
+ randomized_people = matched_people - sum(pinned_by_variant.values())
+ weights = {variant.key: variant.rollout_percentage for variant in pins.split}
+
+ def variant_people(key: str) -> float:
+ weight = weights.get(key)
+ # An unreadable weight counts as the whole split, which gives the lowest share the pins can have.
+ return pinned_by_variant[key] + randomized_people * (1.0 if weight is None else weight / 100)
+
+ def share(key: str) -> float:
+ people = variant_people(key)
+ return pinned_by_variant[key] / people if people else 0.0
+
+ most_pinned_variant = max(pinned_by_variant, key=share)
+ return PinnedShareMeasurement(
+ most_pinned_variant=most_pinned_variant,
+ pinned_people=pinned_by_variant[most_pinned_variant],
+ variant_people=variant_people(most_pinned_variant),
+ matched_people=matched_people,
+ remotely_evaluated_share=traffic.remotely_evaluated_calls / traffic.calls if traffic.calls else None,
+ since=traffic.since,
+ team_timezone=traffic.team_timezone,
+ )
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index d677c8ced8c3..d6014dda2389 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -5,6 +5,7 @@ from posthog.exceptions_capture import capture_exception
from products.experiments.backend.facade.contracts import ExperimentHealthFinding
from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
+from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
@@ -22,6 +23,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
flag_history_not_readable,
split_changed_mid_run,
bucketing_changed_mid_run,
+ release_conditions_changed_mid_run,
flag_variants_mismatch,
forced_variant_release_condition,
no_metric,
diff --git a/products/experiments/backend/health/split_changes.py b/products/experiments/backend/health/split_changes.py
index bef9a76133a4..8c5856c67e01 100644
--- a/products/experiments/backend/health/split_changes.py
+++ b/products/experiments/backend/health/split_changes.py
@@ -40,16 +40,17 @@ class SplitNetEffect:
change_count: int
-# Freezing narrows the release conditions to the exposed people, and unfreezing undoes it. Neither changes
-# the split, but both write the whole `filters` value.
-_SKIPPED_JOB_TYPES = frozenset({"experiment_exposure_frozen", "experiment_exposure_unfrozen"})
+# Freezing narrows the release conditions to the exposed people, and unfreezing undoes it. Both write the whole
+# `filters` value, but neither moves an exposed person to another variant, and the experiment's frozen status
+# already reports them.
+EXPOSURE_FREEZE_JOB_TYPES = frozenset({"experiment_exposure_frozen", "experiment_exposure_unfrozen"})
def split_change_from_activity(
*, changed_at: datetime, job_type: object, before_variants: object, after_variants: object
) -> SplitChange | None:
"""The split change in one activity log row of the flag, or None when the row does not change the split."""
- if job_type in _SKIPPED_JOB_TYPES:
+ if job_type in EXPOSURE_FREEZE_JOB_TYPES:
return None
before = parse_variants(before_variants)
after = parse_variants(after_variants)
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index ff6d76b453a8..acc1c8ac214a 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -4,11 +4,14 @@ from typing import TypedDict
from products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind
from products.experiments.backend.health.context import (
+ ChangeWindow,
+ ConditionsChange,
FlagChange,
FlagHistory,
FlagHistoryNotChecked,
FlagHistoryNotCheckedReason,
HealthContext,
+ ReleaseCondition,
)
# Tests set what their check reads with `dataclasses.replace`, so a new field of the context changes only this file.
@@ -47,6 +50,8 @@ def flag_history(
exposed_before_first: bool | None = True,
exposed_between: Mapping[ExperimentHealthFlagChangeKind, bool | None] | None = None,
not_checked: FlagHistoryNotCheckedReason | None = None,
+ release_conditions: tuple[ReleaseCondition, ...] | None = None,
+ exposed_during_overrides: Mapping[ChangeWindow, bool | None] | None = None,
) -> FlagHistory | FlagHistoryNotChecked:
# Every change counts from the first, as when someone was exposed before it.
if not_checked is not None:
@@ -60,4 +65,8 @@ def flag_history(
counted_since=changes[0].changed_at if changes else None,
exposed_before_first=exposed_before_first,
exposed_between=exposed_between or {},
+ release_conditions=release_conditions
+ if release_conditions is not None
+ else next((change.after for change in reversed(changes) if isinstance(change, ConditionsChange)), ()),
+ exposed_during_overrides=exposed_during_overrides or {},
)
diff --git a/products/experiments/backend/health/test/test_condition_changes.py b/products/experiments/backend/health/test/test_condition_changes.py
new file mode 100644
index 000000000000..6cfe0bdb5c7a
--- /dev/null
+++ b/products/experiments/backend/health/test/test_condition_changes.py
@@ -0,0 +1,160 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from unittest import TestCase
+
+from parameterized import parameterized
+
+from products.experiments.backend.health.condition_changes import (
+ ConditionsChangeShape,
+ conditions_change_from_activity,
+ conditions_net_effect,
+)
+from products.experiments.backend.health.context import ConditionsChange
+
+CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
+PRO = {"key": "plan", "value": ["pro"], "operator": "exact", "type": "person"}
+PRO_OR_TEAM = {"key": "plan", "value": ["pro", "team"], "operator": "exact", "type": "person"}
+EU = {"key": "$geoip_continent_code", "value": ["EU"], "operator": "exact", "type": "person"}
+VARIANTS = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
+
+
+def _condition(
+ *properties: dict[str, Any], rollout: int = 100, variant: str | None = None, **extra: Any
+) -> dict[str, Any]:
+ return {"properties": list(properties), "rollout_percentage": rollout, "variant": variant, **extra}
+
+
+def _change(before: list[dict[str, Any]], after: list[dict[str, Any]], *, minutes: int = 0) -> ConditionsChange:
+ change = conditions_change_from_activity(
+ changed_at=CHANGED_AT + timedelta(minutes=minutes),
+ job_type=None,
+ before_groups=before,
+ after_groups=after,
+ before_aggregation=None,
+ after_aggregation=None,
+ before_variants=VARIANTS,
+ after_variants=VARIANTS,
+ )
+ assert change is not None
+ return change
+
+
+class TestConditionChanges(TestCase):
+ @parameterized.expand(
+ [
+ ("criteria_changed", [_condition(PRO)], [_condition(EU)], None, ConditionsChangeShape.CONDITIONS_CHANGED),
+ (
+ "condition_added",
+ [_condition(PRO)],
+ [_condition(PRO), _condition()],
+ None,
+ ConditionsChangeShape.CONDITIONS_CHANGED,
+ ),
+ (
+ "condition_removed",
+ [_condition(PRO), _condition()],
+ [_condition()],
+ None,
+ ConditionsChangeShape.CONDITIONS_CHANGED,
+ ),
+ (
+ "override_added",
+ [_condition(PRO)],
+ [_condition(PRO, variant="test")],
+ None,
+ ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ ),
+ (
+ "pinned_conditions_reordered",
+ [_condition(PRO, variant="test"), _condition(EU, variant="control")],
+ [_condition(EU, variant="control"), _condition(PRO, variant="test")],
+ None,
+ ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ ),
+ (
+ "condition_without_override_moved_above_a_pinned_one",
+ [_condition(PRO, variant="test"), _condition(EU)],
+ [_condition(EU), _condition(PRO, variant="test")],
+ None,
+ ConditionsChangeShape.VARIANT_OVERRIDE_CHANGED,
+ ),
+ (
+ "filters_of_a_pinned_condition_edited",
+ [_condition(PRO, variant="test"), _condition()],
+ [_condition(PRO_OR_TEAM, variant="test"), _condition()],
+ None,
+ ConditionsChangeShape.CONDITIONS_CHANGED,
+ ),
+ (
+ "override_removed_at_zero_rollout",
+ [_condition(PRO, rollout=0, variant="test")],
+ [_condition(PRO, rollout=0)],
+ None,
+ None,
+ ),
+ (
+ "group_aggregation_made_explicit",
+ [_condition(PRO)],
+ [_condition(PRO, aggregation_group_type_index=0)],
+ None,
+ None,
+ 0,
+ ),
+ (
+ "inherited_group_aggregation_set_to_persons",
+ [_condition(PRO)],
+ [_condition(PRO, aggregation_group_type_index=None)],
+ None,
+ ConditionsChangeShape.CONDITIONS_CHANGED,
+ 0,
+ ),
+ ("rollout_only", [_condition(PRO, rollout=100)], [_condition(PRO, rollout=20)], None, None),
+ ("override_to_no_variant_removed", [_condition(PRO, variant="holdout")], [_condition(PRO)], None, None),
+ ("description_only", [_condition(PRO)], [_condition(PRO, description="Pro users")], None, None),
+ ("filters_reordered", [_condition(PRO, EU)], [_condition(EU, PRO)], None, None),
+ (
+ "unpinned_conditions_reordered",
+ [_condition(PRO), _condition(EU)],
+ [_condition(EU), _condition(PRO)],
+ None,
+ None,
+ ),
+ ("exposure_frozen", [_condition(PRO)], [_condition(PRO, EU)], "experiment_exposure_frozen", None),
+ ("no_conditions_in_the_row", None, None, None, None),
+ ]
+ )
+ def test_conditions_change_from_activity(
+ self,
+ _name: str,
+ before: list[dict[str, Any]] | None,
+ after: list[dict[str, Any]] | None,
+ job_type: str | None,
+ expected: ConditionsChangeShape | None,
+ flag_aggregation: int | None = None,
+ ) -> None:
+ change = conditions_change_from_activity(
+ changed_at=CHANGED_AT,
+ job_type=job_type,
+ before_groups=before,
+ after_groups=after,
+ before_aggregation=flag_aggregation,
+ after_aggregation=flag_aggregation,
+ before_variants=VARIANTS,
+ after_variants=VARIANTS,
+ )
+ effect = conditions_net_effect([change] if change else [], current=None)
+
+ self.assertEqual(effect.shape if effect else None, expected)
+
+ def test_conditions_set_back_are_reverted(self) -> None:
+ effect = conditions_net_effect(
+ [
+ _change([_condition(PRO, variant="test")], [_condition()]),
+ _change([_condition()], [_condition(PRO, variant="test")], minutes=17),
+ ],
+ current=None,
+ )
+
+ assert effect is not None
+ self.assertEqual((effect.shape, effect.change_count), (ConditionsChangeShape.REVERTED, 2))
diff --git a/products/experiments/backend/health/test/test_conditions_changed.py b/products/experiments/backend/health/test/test_conditions_changed.py
new file mode 100644
index 000000000000..bc91d2c47275
--- /dev/null
+++ b/products/experiments/backend/health/test/test_conditions_changed.py
@@ -0,0 +1,368 @@
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from unittest import TestCase
+
+from parameterized import parameterized
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthFlagChangeKind,
+)
+from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
+from products.experiments.backend.health.condition_changes import parse_release_conditions
+from products.experiments.backend.health.context import (
+ ChangeWindow,
+ ConditionsChange,
+ ConditionSetTraffic,
+ FlagHistoryNotCheckedReason,
+ HealthContext,
+ parse_flag_state,
+)
+from products.experiments.backend.health.test.factories import RUNNING_CONTEXT, flag_history
+
+LAUNCHED_AT = datetime(2026, 9, 12, 10, 0, tzinfo=UTC)
+FIRST_CHANGE_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
+FIRST_CHANGE = "September 12, 2026 at 3:04 PM CEST"
+SECOND_CHANGE = "September 12, 2026 at 3:21 PM CEST"
+THIRD_CHANGE = "September 12, 2026 at 3:38 PM CEST"
+EVEN = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
+WITH_HOLDOUT = [*EVEN, {"key": "holdout", "rollout_percentage": 0}]
+PRO = {"properties": [{"key": "plan", "value": ["pro"], "operator": "exact", "type": "person"}]}
+EVERYONE: dict[str, Any] = {"properties": []}
+PRO_PINNED = {**PRO, "variant": "test"}
+PRO_OR_TEAM_PINNED = {
+ "properties": [{"key": "plan", "value": ["pro", "team"], "operator": "exact", "type": "person"}],
+ "variant": "test",
+}
+TEAM = {"properties": [{"key": "plan", "value": ["team"], "operator": "exact", "type": "person"}]}
+EU = {"properties": [{"key": "$geoip_continent_code", "value": ["EU"], "operator": "exact", "type": "person"}]}
+EU_CONTROL = {
+ "properties": [{"key": "$geoip_continent_code", "value": ["EU"], "operator": "exact", "type": "person"}],
+ "variant": "control",
+}
+TRAFFIC_SINCE = "September 13, 2026 at 10:00 AM CEST"
+
+
+def _context(
+ *conditions: list[dict[str, Any]],
+ exposed_between: bool | None = None,
+ not_checked: FlagHistoryNotCheckedReason | None = None,
+ people_by_set: dict[int, int] | None = None,
+ now: list[dict[str, Any]] | None = None,
+ exposed_during_overrides: dict[ChangeWindow, bool | None] | None = None,
+ variants: list[dict[str, Any]] = EVEN,
+) -> HealthContext:
+ variant_keys = [variant["key"] for variant in variants]
+ changes = [
+ ConditionsChange(
+ changed_at=FIRST_CHANGE_AT + timedelta(minutes=17 * index),
+ before=parse_release_conditions(before, variant_keys=variant_keys),
+ after=parse_release_conditions(after, variant_keys=variant_keys),
+ )
+ for index, (before, after) in enumerate(zip(conditions, conditions[1:]))
+ ]
+ return replace(
+ RUNNING_CONTEXT,
+ flag=parse_flag_state(active=True, deleted=False, groups=now or conditions[-1], variants=variants),
+ flag_history=flag_history(
+ changes,
+ launched_at=LAUNCHED_AT,
+ exposed_between={ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS: exposed_between},
+ not_checked=not_checked,
+ release_conditions=None if now is None else parse_release_conditions(now, variant_keys=variant_keys),
+ exposed_during_overrides=exposed_during_overrides,
+ ),
+ condition_set_traffic=None if people_by_set is None else _traffic(people_by_set),
+ )
+
+
+def _traffic(people_by_set: dict[int, int]) -> ConditionSetTraffic:
+ calls = 2 * sum(people_by_set.values())
+ return ConditionSetTraffic(
+ since=datetime(2026, 9, 13, 8, 0, tzinfo=UTC),
+ team_timezone="Europe/Berlin",
+ people_by_set=people_by_set,
+ calls=calls,
+ remotely_evaluated_calls=calls,
+ )
+
+
+INFO = ExperimentHealthFindingSeverity.INFO
+WARNING = ExperimentHealthFindingSeverity.WARNING
+CHANGED_TITLE = "The release conditions changed after launch"
+OVERRIDE_TITLE = "A variant override was removed after launch"
+REORDERED_TITLE = "The order of the release conditions changed after launch"
+OVERRIDE_MOVED = (
+ 'Users it matched got "test" while the override was in place and a random variant after, so those exposed while '
+ 'it was in place may have switched variants. Their exposures under "test" stay in the results.'
+)
+
+
+class TestReleaseConditionsChangedMidRun(TestCase):
+ @parameterized.expand(
+ [
+ (
+ "condition_added",
+ _context([PRO], [PRO, EVERYONE]),
+ ("conditions_changed", INFO, CHANGED_TITLE, "A6"),
+ f"The release conditions changed on {FIRST_CHANGE}, from 1 condition to 2 conditions. People who match "
+ "differently now enter or leave the experiment, and people exposed before the change stay in the "
+ "results. Review the change in the flag history.",
+ ),
+ (
+ "criteria_changed_twice",
+ _context([PRO], [EVERYONE], [{"properties": [{"key": "plan", "value": ["team"], "type": "person"}]}]),
+ ("conditions_changed", INFO, CHANGED_TITLE, "A6"),
+ f"The release conditions changed 2 times between {FIRST_CHANGE} and {SECOND_CHANGE}. People who "
+ "match differently now enter or leave the experiment, and people exposed before the changes stay in "
+ "the results. Review the changes in the flag history.",
+ ),
+ (
+ "override_removed_from_a_large_share",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 300, 2: 2700}),
+ ("variant_override_changed", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} Since '
+ f'{TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" with the override in '
+ f"place (300 of about 1,650). Reset the experiment, or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_from_a_few_percent",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 30, 2: 2970}),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ None,
+ ),
+ (
+ "override_removed_with_too_few_people",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 10, 2: 50}),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} Too few '
+ f"users called the flag since {TRAFFIC_SINCE} to measure how many users Set 1 matches. Reset the "
+ f"experiment, or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_without_a_measurement",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE]),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '
+ "can't measure how many users Set 1 matches. Reset the experiment, or move its start date to "
+ f"{FIRST_CHANGE}.",
+ ),
+ (
+ "override_changed_then_removed",
+ _context(
+ [PRO_PINNED, EVERYONE],
+ [{**PRO, "variant": "control"}, EVERYONE],
+ [PRO, EVERYONE],
+ people_by_set={1: 300, 2: 2700},
+ ),
+ ("variant_override_changed", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "control" of Set 1 was set on {FIRST_CHANGE} and removed on {SECOND_CHANGE}. '
+ 'Users it matched got "control" while the override was in place and a random variant after, so '
+ 'those exposed while it was in place may have switched variants. Their exposures under "control" '
+ f"stay in the results. Since {TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "
+ f'"control" with the override in place (300 of about 1,650). Reset the experiment, or move its start '
+ f"date to {SECOND_CHANGE}.",
+ ),
+ (
+ "override_removed_set_back_and_removed_again",
+ _context(
+ [PRO_PINNED, EVERYONE],
+ [PRO, EVERYONE],
+ [PRO_PINNED, EVERYONE],
+ [PRO, EVERYONE],
+ people_by_set={1: 300, 2: 2700},
+ exposed_during_overrides={
+ ChangeWindow(
+ start=FIRST_CHANGE_AT + timedelta(minutes=17), end=FIRST_CHANGE_AT + timedelta(minutes=34)
+ ): False
+ },
+ ),
+ ("variant_override_changed", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {THIRD_CHANGE}. {OVERRIDE_MOVED} Since '
+ f'{TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" with the override in '
+ f"place (300 of about 1,650). Reset the experiment, or move its start date to {THIRD_CHANGE}.",
+ ),
+ (
+ "override_removed_with_its_rollout_set_to_zero",
+ _context([PRO_PINNED, EVERYONE], [{**PRO, "rollout_percentage": 0}, EVERYONE], people_by_set={2: 3000}),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '
+ f"can't measure how many users the override matched. Reset the experiment, or move its start date to "
+ f"{FIRST_CHANGE}.",
+ ),
+ (
+ "overrides_removed_from_a_small_share_and_a_variant_at_zero_percent",
+ _context(
+ [PRO_PINNED, {**TEAM, "variant": "holdout"}, EVERYONE],
+ [PRO, TEAM, EVERYONE],
+ people_by_set={1: 10, 2: 5, 3: 2985},
+ variants=WITH_HOLDOUT,
+ ),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ None,
+ ),
+ (
+ "override_removed_with_its_condition_next_to_a_small_share",
+ _context(
+ [PRO_PINNED, {**TEAM, "variant": "test"}, EVERYONE],
+ [TEAM, EVERYONE],
+ people_by_set={1: 10, 2: 2990},
+ ),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'A release condition with the variant override "test" was removed on {FIRST_CHANGE}. The variant '
+ f'override "test" of Set 1 was removed on {FIRST_CHANGE}. Users they matched got the override\'s '
+ "variant while it was in place and a random variant after, so those exposed while it was in place "
+ "may have switched variants. Their exposures stay in the results. PostHog can't measure how many "
+ f"users these conditions match. Reset the experiment, or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_with_its_condition",
+ _context([PRO_PINNED, EVERYONE], [EVERYONE]),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'A release condition with the variant override "test" was removed on {FIRST_CHANGE}. '
+ f"{OVERRIDE_MOVED} PostHog can't measure how many users the override matched. Reset the experiment, "
+ f"or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_set_and_removed_again",
+ _context(
+ [PRO, EVERYONE],
+ [PRO_PINNED, EVERYONE],
+ [PRO, EVERYONE],
+ exposed_between=True,
+ people_by_set={1: 300, 2: 2700},
+ ),
+ ("reverted", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was set on {FIRST_CHANGE} and removed on {SECOND_CHANGE}. '
+ f'{OVERRIDE_MOVED} Since {TRAFFIC_SINCE}, Set 1 matched about 18.2% of the users who would get "test" '
+ "with the override in place (300 of about 1,650). Reset the experiment, or move its start date to "
+ f"{SECOND_CHANGE}.",
+ ),
+ (
+ "override_removed_before_a_reorder_that_is_no_change",
+ _context(
+ [PRO_PINNED, EU, EVERYONE],
+ [PRO, EU, EVERYONE],
+ now=[EU, PRO, EVERYONE],
+ people_by_set={1: 30, 2: 300, 3: 2670},
+ ),
+ ("variant_override_changed", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 2 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} Since '
+ f'{TRAFFIC_SINCE}, Set 2 matched about 18.2% of the users who would get "test" with the override in '
+ f"place (300 of about 1,650). Reset the experiment, or move its start date to {FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_with_its_filters_edited",
+ _context([PRO_PINNED, EVERYONE], [TEAM, EVERYONE], people_by_set={2: 3000}),
+ ("variant_override_changed", INFO, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '
+ f"can't measure how many users the override matched. Reset the experiment, or move its start date to "
+ f"{FIRST_CHANGE}.",
+ ),
+ (
+ "override_removed_and_set_back",
+ _context(
+ [PRO_PINNED, EVERYONE],
+ [PRO, EVERYONE],
+ [PRO_PINNED, EVERYONE],
+ exposed_between=True,
+ people_by_set={1: 300, 2: 2700},
+ ),
+ ("reverted", WARNING, OVERRIDE_TITLE, "A7"),
+ f'The variant override "test" of Set 1 changed on {FIRST_CHANGE} and was set back on {SECOND_CHANGE}. '
+ 'Between these changes, users it matches did not get "test", so those exposed then may have '
+ f"switched variants. Their exposures stay in the results. Since {TRAFFIC_SINCE}, Set 1 matched about "
+ '18.2% of the users who would get "test" with the override in place (300 of about 1,650). Reset the '
+ f"experiment, or move its start date to {SECOND_CHANGE}.",
+ ),
+ (
+ "pinned_conditions_reordered",
+ _context([PRO_PINNED, EU_CONTROL, EVERYONE], [EU_CONTROL, PRO_PINNED, EVERYONE]),
+ ("variant_override_changed", INFO, REORDERED_TITLE, "A7"),
+ f"The release conditions changed on {FIRST_CHANGE}, and conditions that serve different variants now "
+ "come in another order. Users who match more than one of them now get the variant of another "
+ "condition, so those exposed before the change may have switched variants. Their exposures stay in "
+ "the results. Review the change in the flag history.",
+ ),
+ (
+ "override_in_place_only_while_nobody_was_exposed",
+ _context(
+ [PRO, EVERYONE],
+ [PRO_PINNED, EVERYONE],
+ [PRO, EVERYONE],
+ [PRO, EVERYONE, EU],
+ people_by_set={1: 300, 2: 2700},
+ exposed_during_overrides={
+ ChangeWindow(start=FIRST_CHANGE_AT, end=FIRST_CHANGE_AT + timedelta(minutes=17)): False
+ },
+ ),
+ ("conditions_changed", INFO, CHANGED_TITLE, "A6"),
+ f"The release conditions changed 3 times between {FIRST_CHANGE} and {THIRD_CHANGE}, from 2 conditions "
+ "to 3 conditions. People who match differently now enter or leave the experiment, and people exposed "
+ "before the changes stay in the results. Review the changes in the flag history.",
+ ),
+ (
+ "filters_of_a_pinned_condition_edited",
+ _context([PRO_PINNED, EVERYONE], [PRO_OR_TEAM_PINNED, EVERYONE]),
+ ("conditions_changed", INFO, CHANGED_TITLE, "A6"),
+ f"The release conditions changed on {FIRST_CHANGE}. People who match differently now enter or leave "
+ "the experiment, and people exposed before the change stay in the results. Set 1 has the variant "
+ 'override "test", so users who now match it or no longer match it may also have switched variants. '
+ "Review the change in the flag history.",
+ ),
+ (
+ "set_back_with_exposures_in_between",
+ _context([PRO], [EVERYONE], [PRO], exposed_between=True),
+ ("reverted", INFO, "The release conditions changed and were set back", "A6"),
+ f"The release conditions changed on {FIRST_CHANGE} and were set back on {SECOND_CHANGE}. People "
+ "exposed in between were targeted by other conditions, and they stay in the results. Review the "
+ "changes in the flag history.",
+ ),
+ (
+ "not_checked",
+ _context([PRO], [EVERYONE], not_checked=FlagHistoryNotCheckedReason.NO_QUERY_ACCESS),
+ ("not_checked", INFO, "Release condition changes were not checked", "A6"),
+ f"The release conditions changed on {FIRST_CHANGE}. This check can't tell whether anyone was exposed "
+ "before the change, because this API key can't query events. Add the query:read scope to this API "
+ "key, or review the change in the flag history.",
+ ),
+ ]
+ )
+ def test_reports_release_condition_changes(
+ self,
+ _name: str,
+ ctx: HealthContext,
+ expected: tuple[str, ExperimentHealthFindingSeverity, str, str],
+ expected_detail: str | None,
+ ) -> None:
+ finding = release_conditions_changed_mid_run(ctx)
+
+ assert finding is not None
+ self.assertEqual(finding.code, ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN)
+ self.assertEqual((finding.subcode, finding.severity, finding.title, finding.diagnostic_ref), expected)
+ if expected_detail is not None:
+ self.assertEqual(finding.detail, expected_detail)
+
+ @parameterized.expand(
+ [
+ ("no_change", _context([PRO])),
+ ("set_back_with_nobody_in_between", _context([PRO], [EVERYONE], [PRO], exposed_between=False)),
+ ("no_history_access", _context([PRO], not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS)),
+ # The forced-variant check reports an override in place, with the time it was set.
+ ("override_added", _context([PRO, EVERYONE], [PRO_PINNED, EVERYONE])),
+ ("pinned_condition_added", _context([EVERYONE], [PRO_PINNED, EVERYONE])),
+ (
+ "override_changed_to_another_variant",
+ _context([PRO_PINNED, EVERYONE], [{**PRO, "variant": "control"}, EVERYONE]),
+ ),
+ (
+ "override_removed_from_under_one_percent",
+ _context([PRO_PINNED, EVERYONE], [PRO, EVERYONE], people_by_set={1: 10, 2: 2990}),
+ ),
+ ]
+ )
+ def test_reports_nothing(self, _name: str, ctx: HealthContext) -> None:
+ self.assertIsNone(release_conditions_changed_mid_run(ctx))
diff --git a/products/experiments/backend/health/test/test_exposure_gate.py b/products/experiments/backend/health/test/test_exposure_gate.py
index 037e9612448d..bec74a414bcd 100644
--- a/products/experiments/backend/health/test/test_exposure_gate.py
+++ b/products/experiments/backend/health/test/test_exposure_gate.py
@@ -20,6 +20,8 @@ from products.experiments.backend.facade.contracts import ExperimentHealthFlagCh
from products.experiments.backend.health.context import (
BucketingChange,
BucketingSetting,
+ ChangeWindow,
+ ConditionsChange,
FlagHistory,
FlagHistoryNotChecked,
FlagHistoryNotCheckedReason,
@@ -27,6 +29,7 @@ from products.experiments.backend.health.context import (
)
from products.experiments.backend.health.exposure_gate import (
MAX_EXPOSURE_QUERIES,
+ MAX_OVERRIDE_WINDOWS,
load_flag_history,
query_any_exposure,
)
@@ -49,6 +52,9 @@ _BY_ACTION: dict[str, Any] = {"exposure_config": {"kind": "ActionsNode", "id": 1
_UNREADABLE: dict[str, Any] = {"exposure_config": {"kind": "ExperimentEventExposureConfig"}}
+_EVERYONE: dict[str, Any] = {"properties": [], "rollout_percentage": 100}
+
+
def _split(control: int, test: int) -> list[dict[str, Any]]:
return [{"key": "control", "rollout_percentage": control}, {"key": "test", "rollout_percentage": test}]
@@ -146,12 +152,16 @@ class TestLoadFlagHistory(BaseTest):
def test_every_kind_of_change_shares_one_walk(self) -> None:
first_at = self.launched_at + timedelta(days=1)
changed_at = [first_at, first_at + timedelta(hours=1), first_at + timedelta(hours=2)]
- self._log_change(changed_at[0], filters=(_filters(_split(50, 50)), _filters(_split(55, 45))))
+ with_a_second_condition = {
+ **_filters(_split(50, 50)),
+ "groups": [{"properties": [], "rollout_percentage": 100}, {"properties": [], "rollout_percentage": 50}],
+ }
+ self._log_change(changed_at[0], filters=(_filters(_split(50, 50)), with_a_second_condition))
self._log_change(changed_at[1], bucketing_identifier=("distinct_id", "device_id"))
self._log_change(
changed_at[2],
filters=(
- {**_filters(_split(55, 45)), "aggregation_group_type_index": None},
+ {**_filters(_split(50, 50)), "aggregation_group_type_index": None},
{**_filters(_split(60, 40)), "aggregation_group_type_index": 0},
),
)
@@ -159,8 +169,8 @@ class TestLoadFlagHistory(BaseTest):
history, query = self._load([False, True])
assert isinstance(history, FlagHistory)
- self.assertEqual([change.changed_at for change in history.of_kind(SplitChange)], [changed_at[0], changed_at[2]])
- self.assertEqual([change.changed_at for change in history.counted(SplitChange)], changed_at[2:])
+ self.assertEqual([change.changed_at for change in history.of_kind(ConditionsChange)], changed_at[:1])
+ self.assertEqual(history.counted(ConditionsChange), ())
self.assertEqual(
[(change.changed_at, change.setting) for change in history.counted(BucketingChange)],
[
@@ -168,6 +178,7 @@ class TestLoadFlagHistory(BaseTest):
(changed_at[2], BucketingSetting.AGGREGATION),
],
)
+ self.assertEqual([change.changed_at for change in history.counted(SplitChange)], changed_at[2:])
self.assertEqual(query.call_count, 2)
def test_each_window_asks_with_the_key_the_flag_had_then(self) -> None:
@@ -236,6 +247,67 @@ class TestLoadFlagHistory(BaseTest):
self.assertIs(history.exposed_before_first, True)
self.assertEqual(query.call_count, 1)
+ @parameterized.expand([("too_many_saves", 101, 1), ("too_many_conditions", 1, 101)])
+ def test_a_history_past_the_bounds_is_not_read(self, reason: str, saves: int, conditions: int) -> None:
+ first_at = self.launched_at + timedelta(days=1)
+ filters = [
+ {**_filters(_split(60, 40) if index % 2 else _split(50, 50)), "groups": [_EVERYONE] * conditions}
+ for index in range(saves + 1)
+ ]
+ for index, (before, after) in enumerate(zip(filters, filters[1:])):
+ self._log_change(first_at + timedelta(minutes=index), filters=(before, after))
+
+ history, query = self._load([True])
+
+ assert isinstance(history, FlagHistoryNotChecked)
+ self.assertEqual((history.reason, history.changes), (FlagHistoryNotCheckedReason(reason), ()))
+ self.assertEqual(query.call_count, 0)
+
+ def test_an_override_set_and_removed_asks_whether_anyone_was_exposed_while_it_was_in_place(self) -> None:
+ first_at = self.launched_at + timedelta(days=1)
+ changed_at = [first_at, first_at + timedelta(hours=1), first_at + timedelta(hours=2)]
+ pro = {"properties": [{"key": "plan", "value": ["pro"], "operator": "exact", "type": "person"}]}
+ groups = [[_EVERYONE], [{**_EVERYONE, "variant": "test"}], [_EVERYONE], [_EVERYONE, pro]]
+ for at, before, after in zip(changed_at, groups, groups[1:]):
+ self._log_change(
+ at,
+ filters=({**_filters(_split(50, 50)), "groups": before}, {**_filters(_split(50, 50)), "groups": after}),
+ )
+
+ history, query = self._load([True, False])
+
+ assert isinstance(history, FlagHistory)
+ self.assertEqual(
+ history.exposed_during_overrides, {ChangeWindow(start=changed_at[0], end=changed_at[1]): False}
+ )
+ self.assertEqual(query.call_count, 2)
+
+ def test_override_windows_past_the_bound_are_not_asked(self) -> None:
+ first_at = self.launched_at + timedelta(days=1)
+ unpinned: list[dict[str, Any]] = [
+ {"properties": [{"key": "plan", "value": [f"plan {index}"], "operator": "exact", "type": "person"}]}
+ for index in range(MAX_OVERRIDE_WINDOWS + 1)
+ ]
+ groups = [unpinned]
+ for pinned_index in range(len(unpinned)):
+ groups.append(
+ [
+ {**condition, "variant": "test"} if index == pinned_index else condition
+ for index, condition in enumerate(unpinned)
+ ]
+ )
+ groups.append(unpinned)
+ for minute, (before, after) in enumerate(zip(groups, groups[1:])):
+ self._log_change(
+ first_at + timedelta(minutes=minute),
+ filters=({**_filters(_split(50, 50)), "groups": before}, {**_filters(_split(50, 50)), "groups": after}),
+ )
+
+ history, _ = self._load(lambda **kwargs: kwargs["window_start"] == self.launched_at)
+
+ assert isinstance(history, FlagHistory)
+ self.assertEqual(len(history.exposed_during_overrides), MAX_OVERRIDE_WINDOWS)
+
@parameterized.expand(
[
("exposure_filters", _BY_EVENT_PROPERTY, True, True, FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),
@@ -304,19 +376,36 @@ class TestLoadFlagHistory(BaseTest):
@parameterized.expand(
[
- ("same_split", _split(50, 50)),
- ("with_a_variant_left_at_zero", [*_split(50, 50), {"key": "blue", "rollout_percentage": 0}]),
+ ("same_split", _filters(_split(50, 50)), [ExperimentHealthFlagChangeKind.SPLIT]),
+ (
+ "with_a_variant_left_at_zero",
+ _filters([*_split(50, 50), {"key": "blue", "rollout_percentage": 0}]),
+ [ExperimentHealthFlagChangeKind.SPLIT],
+ ),
+ # Both kinds ask about the same window, and the second must not query again.
+ (
+ "with_the_conditions_set_back_too",
+ _filters(_split(50, 50)),
+ [ExperimentHealthFlagChangeKind.SPLIT, ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS],
+ ),
]
)
def test_revert_asks_whether_anyone_was_exposed_in_between(
- self, _name: str, set_back_to: list[dict[str, Any]]
+ self, _name: str, set_back_to: dict[str, Any], set_back_kinds: list[ExperimentHealthFlagChangeKind]
) -> None:
- changed_at = self._log_split_changes([_split(50, 50), _split(0, 100), set_back_to])
+ first_at = self.launched_at + timedelta(days=1)
+ changed_at = [first_at, first_at + timedelta(hours=1)]
+ changed = _filters(_split(0, 100))
+ if ExperimentHealthFlagChangeKind.RELEASE_CONDITIONS in set_back_kinds:
+ changed["groups"] = [{"properties": [], "rollout_percentage": 100, "variant": "test"}]
+ self._log_change(changed_at[0], filters=(_filters(_split(50, 50)), changed))
+ self._log_change(changed_at[1], filters=(changed, set_back_to))
history, query = self._load([True, False])
assert isinstance(history, FlagHistory)
- self.assertIs(history.exposed_between.get(ExperimentHealthFlagChangeKind.SPLIT), False)
+ self.assertEqual(history.exposed_between, dict.fromkeys(set_back_kinds, False))
+ self.assertEqual(query.call_count, 2)
self.assertEqual(
query.call_args.kwargs,
{
diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.py
index 57f9fd95cac2..cb48d6e5cd12 100644
--- a/products/experiments/backend/health/test/test_forced_variant.py
+++ b/products/experiments/backend/health/test/test_forced_variant.py
@@ -7,12 +7,25 @@ from unittest import TestCase
from parameterized import parameterized
from products.experiments.backend.facade.contracts import ExperimentHealthFindingSeverity
-from products.experiments.backend.health.checks.forced_variant import (
- forced_variant_release_condition,
- needs_condition_set_traffic,
+from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
+from products.experiments.backend.health.condition_changes import parse_release_conditions
+from products.experiments.backend.health.condition_set_traffic import needs_condition_set_traffic
+from products.experiments.backend.health.context import (
+ ConditionsChange,
+ ConditionSetTraffic,
+ FlagHistory,
+ FlagHistoryNotChecked,
+ HealthContext,
+ parse_flag_state,
+)
+from products.experiments.backend.health.test.factories import (
+ DRAFT,
+ ENDED,
+ RUNNING,
+ RUNNING_CONTEXT,
+ Status,
+ flag_history,
)
-from products.experiments.backend.health.context import ConditionSetTraffic, HealthContext, parse_flag_state
-from products.experiments.backend.health.test.factories import DRAFT, ENDED, RUNNING, RUNNING_CONTEXT, Status
EVEN_SPLIT = [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
SHIPPED_SPLIT = [{"key": "control", "rollout_percentage": 0}, {"key": "test", "rollout_percentage": 100}]
@@ -22,11 +35,17 @@ STAFF: dict[str, Any] = {
"properties": [{"key": "email", "value": "@example.com", "operator": "icontains", "type": "person"}],
"rollout_percentage": 100,
}
+STAFF_AT_ONE_DOMAIN: dict[str, Any] = {
+ "properties": [{"key": "email", "value": "[email protected]", "operator": "icontains", "type": "person"}],
+ "rollout_percentage": 100,
+}
IOS: dict[str, Any] = {
"properties": [{"key": "$device_type", "value": "iOS", "operator": "exact", "type": "person"}],
"rollout_percentage": 100,
}
+CHANGED_AT = datetime(2026, 9, 12, 13, 4, tzinfo=UTC)
+
def _context(
groups: list[dict[str, Any]],
@@ -37,16 +56,30 @@ def _context(
early_exit: bool = False,
traffic: ConditionSetTraffic | None = None,
excluded: frozenset[str] = frozenset(),
+ groups_at_launch: list[dict[str, Any]] | None = None,
) -> HealthContext:
return replace(
RUNNING_CONTEXT,
**status,
flag=parse_flag_state(active=True, deleted=deleted, groups=groups, variants=variants, early_exit=early_exit),
+ flag_history=None if groups_at_launch is None else _history(groups_at_launch, groups, variants),
condition_set_traffic=traffic,
excluded_variant_keys=excluded,
)
+def _history(
+ before: list[dict[str, Any]], after: list[dict[str, Any]], variants: list[dict[str, Any]]
+) -> FlagHistory | FlagHistoryNotChecked:
+ variant_keys = [variant["key"] for variant in variants]
+ change = ConditionsChange(
+ changed_at=CHANGED_AT,
+ before=parse_release_conditions(before, variant_keys=variant_keys),
+ after=parse_release_conditions(after, variant_keys=variant_keys),
+ )
+ return flag_history([change], launched_at=datetime(2026, 9, 12, 10, 0, tzinfo=UTC))
+
+
def _traffic(people_by_set: dict[int, int], *, remotely_evaluated: float = 1.0) -> ConditionSetTraffic:
calls = 2 * sum(people_by_set.values())
return ConditionSetTraffic(
@@ -273,7 +306,41 @@ class TestForcedVariantReleaseCondition(TestCase):
_context([{**STAFF, "variant": "holdout"}, EVERYONE], variants=SPLIT_WITH_HOLDOUT),
False,
),
+ (
+ "an_override_removed_after_launch",
+ _context([STAFF, EVERYONE], groups_at_launch=[{**STAFF, "variant": "test"}, EVERYONE]),
+ True,
+ ),
]
)
def test_needs_condition_set_traffic(self, _name: str, ctx: HealthContext, expected: bool) -> None:
self.assertIs(needs_condition_set_traffic(ctx), expected)
+
+ @parameterized.expand(
+ [
+ ("set_after_launch", [STAFF, EVERYONE], True),
+ ("changed_after_launch", [{**STAFF, "variant": "control"}, EVERYONE], True),
+ ("in_place_at_launch", [{**STAFF, "variant": "test"}, EVERYONE, IOS], False),
+ ("filters_edited_after_launch", [{**STAFF_AT_ONE_DOMAIN, "variant": "test"}, EVERYONE], False),
+ ]
+ )
+ def test_an_override_set_after_launch_carries_its_date(
+ self, _name: str, groups_at_launch: list[dict[str, Any]], set_after_launch: bool
+ ) -> None:
+ finding = forced_variant_release_condition(
+ _context([{**STAFF, "variant": "test"}, EVERYONE], groups_at_launch=groups_at_launch)
+ )
+
+ assert finding is not None
+ self.assertEqual(
+ "The override of Set 1 was set on September 12, 2026 at 3:04 PM CEST, after launch, so users it matches "
+ "who were exposed before may have switched variants." in finding.detail,
+ set_after_launch,
+ )
+ self.assertTrue(
+ finding.detail.endswith(
+ "Then move the experiment's start date to after the removal."
+ if set_after_launch
+ else "Then reset the experiment, or move its start date to after the removal."
+ )
+ )
diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.py
index 7d38fbbb57fc..d90475a50a12 100644
--- a/products/experiments/backend/health/test/test_split_changed.py
+++ b/products/experiments/backend/health/test/test_split_changed.py
@@ -305,12 +305,20 @@ class TestSplitChangedMidRun(TestCase):
(
"no_history_access",
_context(EVEN, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),
- "This check can't see whether the flag's variant split, key or bucketing changed after launch, because "
- "you can't view the linked feature flag's history. Viewing the history needs viewer access to the "
- "flag and, with an API key, the activity_log:read scope.",
+ "This check can't see whether the flag's variant split, key, bucketing or release conditions changed "
+ "after launch, because you can't view the linked feature flag's history. Viewing the history needs "
+ "viewer access to the flag and, with an API key, the activity_log:read scope.",
(),
"The flag history was not checked",
),
+ (
+ "too_many_saves",
+ _context(EVEN, not_checked=FlagHistoryNotCheckedReason.TOO_MANY_SAVES),
+ "This check can't see whether the flag's variant split, key, bucketing or release conditions changed "
+ "after launch, because the flag was saved more than 100 times since launch. Review the flag history.",
+ (ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),
+ "The flag history was not checked",
+ ),
]
)
def test_reports_split_changes_that_were_not_checked(
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index 0f8b4d228876..e9512eb4b00a 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -3168,7 +3168,7 @@ _FLAG_CHANGE_KIND_HELP = (
class ExperimentHealthInputsFlagChangeSerializer(serializers.Serializer):
kind = serializers.ChoiceField(choices=ExperimentHealthFlagChangeKind.choices, help_text=_FLAG_CHANGE_KIND_HELP)
changed_at = serializers.DateTimeField(help_text="When the flag changed.")
- changed = serializers.CharField(help_text="What changed: 'Split', or the name of the setting, such as 'Key'.")
+ changed = serializers.CharField(help_text="What changed, as the staff view names it, such as 'Split' or 'Key'.")
before = serializers.CharField(
help_text="The value before the change as text, such as 'control 50%, test 50%' or 'device_id'. 'none' when unset."
)
@@ -3205,7 +3205,8 @@ class ExperimentHealthInputsFlagHistorySerializer(serializers.Serializer):
many=True,
help_text=(
"Every change since launch that can move exposed people, of every kind, in time order. A bucketing change "
- "is one per setting and save. Empty for 'no_history_access', which does not read them."
+ "is one per setting and save. A rollout change alone is no release-condition change. Empty for "
+ "'no_history_access', which does not read them."
),
)
exposed_before_first = serializers.BooleanField(
@@ -3249,8 +3250,8 @@ class ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):
denied_event_properties = serializers.ListField(
child=serializers.CharField(),
help_text=(
- "Event properties the reader is denied. The forced-variant check skips its scan when one of them is a "
- "flag call property it reads."
+ "Event properties the reader is denied. The forced-variant and release-condition checks skip their scan "
+ "when one of them is a flag call property it reads."
),
)
@@ -3343,7 +3344,8 @@ class ExperimentHealthInputsSerializer(serializers.Serializer):
help_text=(
"Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the "
"experiment is launched and some but not all release conditions pin a variant that has a share of the "
- "split and is not excluded from the analysis."
+ "split and is not excluded from the analysis. The release-condition check needs it when such an "
+ "override was removed after launch."
),
)
condition_set_traffic = ExperimentHealthInputsConditionSetTrafficSerializer(
diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py
index 995f4701cc4e..205c63175c47 100644
--- a/products/experiments/backend/test/test_presentation_api.py
+++ b/products/experiments/backend/test/test_presentation_api.py
@@ -89,6 +89,17 @@ _REBALANCED = {
},
}
}
+_WITH_A_SECOND_CONDITION = {
+ "filters": {
+ "groups": [
+ {"properties": [], "rollout_percentage": 100},
+ {"properties": [{"key": "plan", "value": ["pro"], "operator": "exact", "type": "person"}]},
+ ],
+ "multivariate": {
+ "variants": [{"key": "control", "rollout_percentage": 50}, {"key": "test", "rollout_percentage": 50}]
+ },
+ }
+}
_HEALTH_TRAFFIC = ConditionSetTraffic(
since=datetime(2026, 10, 12, tzinfo=UTC),
team_timezone="UTC",
@@ -760,6 +771,13 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):
[_HEALTH_PAUSED, ("bucketing_changed_mid_run", "key_changed")],
(timedelta(days=1), {"key": "health-findings-renamed"}),
),
+ (
+ "condition_added_after_launch",
+ {"metrics": [_HEALTH_METRIC]},
+ None,
+ [_HEALTH_PAUSED, ("release_conditions_changed_mid_run", "conditions_changed")],
+ (timedelta(days=1), _WITH_A_SECOND_CONDITION),
+ ),
]
)
def test_health_reports_findings(
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 7169893c909e..3fd756940ce2 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2275,6 +2275,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
*/
export type ExperimentHealthFindingCodeEnumApi =
(typeof ExperimentHealthFindingCodeEnumApi)[keyof typeof ExperimentHealthFindingCodeEnumApi]
@@ -2292,6 +2293,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
FlagVariantsMismatch: 'flag_variants_mismatch',
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
} as const
/**
@@ -2351,7 +2353,8 @@ export interface ExperimentHealthFindingApi {
* * `forced_variant_release_condition` - Forced Variant Release Condition
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
- * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run */
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run */
code: ExperimentHealthFindingCodeEnumApi
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -3066,6 +3069,7 @@ export interface ExperimentHealthInputsFlagApi {
/**
* * `split` - Split
* * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition
*/
export type ExperimentHealthFlagChangeKindEnumApi =
(typeof ExperimentHealthFlagChangeKindEnumApi)[keyof typeof ExperimentHealthFlagChangeKindEnumApi]
@@ -3073,17 +3077,19 @@ export type ExperimentHealthFlagChangeKindEnumApi =
export const ExperimentHealthFlagChangeKindEnumApi = {
Split: 'split',
Bucketing: 'bucketing',
+ ReleaseConditions: 'release_conditions',
} as const
export interface ExperimentHealthInputsFlagChangeApi {
/** The kind of change. A bucketing change is a change of a setting that decides the variant draw.
*
* * `split` - Split
- * * `bucketing` - Bucketing */
+ * * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition */
kind: ExperimentHealthFlagChangeKindEnumApi
/** When the flag changed. */
changed_at: string
- /** What changed: 'Split', or the name of the setting, such as 'Key'. */
+ /** What changed, as the staff view names it, such as 'Split' or 'Key'. */
changed: string
/** The value before the change as text, such as 'control 50%, test 50%' or 'device_id'. 'none' when unset. */
before: string
@@ -3097,7 +3103,8 @@ export interface ExperimentHealthInputsExposedBetweenApi {
/** The kind of change. A bucketing change is a change of a setting that decides the variant draw.
*
* * `split` - Split
- * * `bucketing` - Bucketing */
+ * * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition */
kind: ExperimentHealthFlagChangeKindEnumApi
/** What was asked, as the staff view names it. */
question: string
@@ -3120,7 +3127,7 @@ export interface ExperimentHealthInputsFlagHistoryApi {
launched_at: string
/** The project's time zone, which the findings use for times. */
team_timezone: string
- /** Every change since launch that can move exposed people, of every kind, in time order. A bucketing change is one per setting and save. Empty for 'no_history_access', which does not read them. */
+ /** Every change since launch that can move exposed people, of every kind, in time order. A bucketing change is one per setting and save. A rollout change alone is no release-condition change. Empty for 'no_history_access', which does not read them. */
changes: ExperimentHealthInputsFlagChangeApi[]
/**
* Whether anyone was exposed before the first counted change. Null when unknown or not checked.
@@ -3176,7 +3183,7 @@ export interface ExperimentHealthInputsReaderAccessApi {
can_read_flag_history: boolean
/** Whether the reader may query events, which the flag-change and forced-variant checks need. */
can_query_events: boolean
- /** Event properties the reader is denied. The forced-variant check skips its scan when one of them is a flag call property it reads. */
+ /** Event properties the reader is denied. The forced-variant and release-condition checks skip their scan when one of them is a flag call property it reads. */
denied_event_properties: string[]
}
@@ -3450,7 +3457,7 @@ export interface ExperimentHealthInputsApi {
flag: ExperimentHealthInputsFlagApi | null
/** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */
flag_history: ExperimentHealthInputsFlagHistoryApi | null
- /** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis.
+ /** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check needs it when such an override was removed after launch.
*
* * `read` - Read
* * `not_needed` - Not Needed
diff --git a/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx b/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx
index 0e12e502218b..29b2c71913f3 100644
--- a/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx
+++ b/products/experiments/frontend/health/debug/HealthDebugFlagTab.tsx
@@ -85,7 +85,7 @@ export function HealthDebugFlagTab(): JSX.Element {
</HealthDebugSection>
<HealthDebugSection
title="Changes since launch"
- description="What the split and bucketing checks read from the flag's history."
+ description="What the split, bucketing and release-condition checks read from the flag's history."
>
<SetupFactsTable facts={flagHistoryFacts} />
{flagChanges.length > 0 && (
@@ -114,7 +114,7 @@ export function HealthDebugFlagTab(): JSX.Element {
</HealthDebugSection>
<HealthDebugSection
title="Calls by release condition"
- description="The flag's recent calls that the forced-variant check grades pinned conditions by."
+ description="The flag's recent calls that the forced-variant and release-condition checks grade overrides by."
>
<SetupFactsTable facts={trafficFacts} />
</HealthDebugSection>
diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.ts
index b76daccc2771..674fff8da410 100644
--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts
+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts
@@ -29,7 +29,7 @@ export interface HealthDebugReleaseCondition {
export interface HealthDebugFlagChange {
changedAt: string
- /** What changed: the split, or a setting that decides the variant draw. */
+ /** What changed: the split, a setting that decides the variant draw, or the release conditions. */
changed: string
before: string
after: string
@@ -213,7 +213,7 @@ export function buildFlagChanges(inputs: ExperimentHealthInputsApi | null): Heal
const TRAFFIC_STATUS_LABELS: Record<ExperimentHealthInputsTrafficStatusEnumApi, string> = {
read: 'Read',
- not_needed: 'Not read: the forced-variant check does not need it',
+ not_needed: 'Not read: no check needs it',
no_query_access: 'Not read: you may not query events',
denied_properties: 'Not read: you are denied a flag call property that the scan reads',
no_flag_version: "Not read: the flag has no version, so its calls can't be matched to it",
@@ -227,7 +227,7 @@ export function buildTrafficFacts(state: HealthInputsState): SetupFact[] {
{
label: 'Status',
value: serverValue(state, (inputs) => TRAFFIC_STATUS_LABELS[inputs.condition_set_traffic_status]),
- help: 'The forced-variant check reads the calls when the experiment is launched and some, but not all, release conditions pin a variant that has a share of the split and is not excluded from the analysis.',
+ help: 'The forced-variant check reads the calls when the experiment is launched and some, but not all, release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check reads them when such an override was removed after launch.',
},
...(traffic
? [
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 2c2dcbbf673e..35aabe17efbe 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -190,6 +190,21 @@ function bucketingChangedNote(finding: ExperimentHealthFindingApi): string {
: `${changed} ${exposedBeforeFirst(finding)}`
}
+function conditionsChangedNote(finding: ExperimentHealthFindingApi): string {
+ const { evidence } = finding
+ const changed = `${evidence.change_count} changes, from ${evidence.conditions_before} to ${evidence.conditions_after} conditions.`
+ if (finding.subcode === 'not_checked') {
+ return `Not checked: ${evidence.reason}. ${changed}`
+ }
+ const pinnedShare = share(evidence.pinned_share)
+ const removedOverride = pinnedShare
+ ? `Removed override share ${pinnedShare}.`
+ : typeof evidence.reason === 'string'
+ ? `Removed override share unknown: ${evidence.reason}.`
+ : null
+ return [changed, exposedBeforeFirst(finding), removedOverride].filter(Boolean).join(' ')
+}
+
function forcedVariantNote(finding: ExperimentHealthFindingApi): string {
const { evidence } = finding
const parts = [`Pinned sets: ${evidence.pinned_condition_sets}, to ${evidence.pinned_variant_keys}.`]
@@ -292,6 +307,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChec
flagStateCheck(input),
serverOnlyCheck(input, 'split_changed_mid_run', splitChangedNote),
serverOnlyCheck(input, 'bucketing_changed_mid_run', bucketingChangedNote),
+ serverOnlyCheck(input, 'release_conditions_changed_mid_run', conditionsChangedNote),
serverOnlyCheck(input, 'flag_variants_mismatch', null),
serverOnlyCheck(input, 'forced_variant_release_condition', forcedVariantNote),
noMetricCheck(input),
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index d9e77759b50e..372b6652c090 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -25,6 +25,7 @@ export type ExperimentHealthFindingCode =
| 'flag_variants_mismatch'
| 'split_changed_mid_run'
| 'bucketing_changed_mid_run'
+ | 'release_conditions_changed_mid_run'
export interface ExperimentHealthFinding {
code: ExperimentHealthFindingCode
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index f702da3dee17..4e630b940199 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -44673,6 +44673,7 @@ export namespace Schemas {
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
* * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
*/
export type ExperimentHealthFindingCodeEnum = typeof ExperimentHealthFindingCodeEnum[keyof typeof ExperimentHealthFindingCodeEnum];
@@ -44690,6 +44691,7 @@ export namespace Schemas {
FlagVariantsMismatch: 'flag_variants_mismatch',
SplitChangedMidRun: 'split_changed_mid_run',
BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
} as const;
/**
@@ -44749,7 +44751,8 @@ export namespace Schemas {
* * `forced_variant_release_condition` - Forced Variant Release Condition
* * `flag_variants_mismatch` - Flag Variants Mismatch
* * `split_changed_mid_run` - Split Changed Mid Run
- * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run */
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run */
code: ExperimentHealthFindingCodeEnum;
/**
* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.
@@ -45318,6 +45321,7 @@ export namespace Schemas {
/**
* * `split` - Split
* * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition
*/
export type ExperimentHealthFlagChangeKindEnum = typeof ExperimentHealthFlagChangeKindEnum[keyof typeof ExperimentHealthFlagChangeKindEnum];
@@ -45325,6 +45329,7 @@ export namespace Schemas {
export const ExperimentHealthFlagChangeKindEnum = {
Split: 'split',
Bucketing: 'bucketing',
+ ReleaseConditions: 'release_conditions',
} as const;
export interface ExperimentHealthInputsReleaseCondition {
@@ -45379,11 +45384,12 @@ export namespace Schemas {
/** The kind of change. A bucketing change is a change of a setting that decides the variant draw.
*
* * `split` - Split
- * * `bucketing` - Bucketing */
+ * * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition */
kind: ExperimentHealthFlagChangeKindEnum;
/** When the flag changed. */
changed_at: string;
- /** What changed: 'Split', or the name of the setting, such as 'Key'. */
+ /** What changed, as the staff view names it, such as 'Split' or 'Key'. */
changed: string;
/** The value before the change as text, such as 'control 50%, test 50%' or 'device_id'. 'none' when unset. */
before: string;
@@ -45397,7 +45403,8 @@ export namespace Schemas {
/** The kind of change. A bucketing change is a change of a setting that decides the variant draw.
*
* * `split` - Split
- * * `bucketing` - Bucketing */
+ * * `bucketing` - Bucketing
+ * * `release_conditions` - Release-condition */
kind: ExperimentHealthFlagChangeKindEnum;
/** What was asked, as the staff view names it. */
question: string;
@@ -45420,7 +45427,7 @@ export namespace Schemas {
launched_at: string;
/** The project's time zone, which the findings use for times. */
team_timezone: string;
- /** Every change since launch that can move exposed people, of every kind, in time order. A bucketing change is one per setting and save. Empty for 'no_history_access', which does not read them. */
+ /** Every change since launch that can move exposed people, of every kind, in time order. A bucketing change is one per setting and save. A rollout change alone is no release-condition change. Empty for 'no_history_access', which does not read them. */
changes: ExperimentHealthInputsFlagChange[];
/**
* Whether anyone was exposed before the first counted change. Null when unknown or not checked.
@@ -45476,7 +45483,7 @@ export namespace Schemas {
can_read_flag_history: boolean;
/** Whether the reader may query events, which the flag-change and forced-variant checks need. */
can_query_events: boolean;
- /** Event properties the reader is denied. The forced-variant check skips its scan when one of them is a flag call property it reads. */
+ /** Event properties the reader is denied. The forced-variant and release-condition checks skip their scan when one of them is a flag call property it reads. */
denied_event_properties: string[];
}
@@ -45751,7 +45758,7 @@ export namespace Schemas {
flag: ExperimentHealthInputsFlag | null;
/** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */
flag_history: ExperimentHealthInputsFlagHistory | null;
- /** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis.
+ /** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check needs it when such an override was removed after launch.
*
* * `read` - Read
* * `not_needed` - Not Needed